def __init__(self, scope: core.App, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
pass_through_lambda = _lambda.Function(
self,
'PassThroughLambda',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='pass_through_lambda.handler')
loop_count_lambda = _lambda.Function(
self,
'LoopCountLambda',
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.Code.asset('lambda'),
handler='loop_count_lambda.handler')
start_state_machine = sfn.Task(
self,
"Start CodeBuild Lambda",
task=sfn_tasks.InvokeFunction(pass_through_lambda))
wait_x = sfn.Wait(
self,
"Wait X Seconds",
time=sfn.WaitTime.seconds_path('$.wait_time'),
)
get_state_machine_status = sfn.Task(
self,
"Get Build Status",
task=sfn_tasks.InvokeFunction(loop_count_lambda))
is_complete = sfn.Choice(self, "Job Complete?")
state_machine_failed = sfn.Fail(self,
"Build Failed",
cause="AWS Batch Job Failed",
error="DescribeJob returned FAILED")
state_machine_success = sfn.Pass(self, "Build Successs")
definition = start_state_machine\
.next(wait_x)\
.next(get_state_machine_status)\
.next(is_complete
.when(sfn.Condition.string_equals(
"$.status", "FAILED"), state_machine_failed)
.when(sfn.Condition.string_equals(
"$.status", "SUCCEEDED"), state_machine_success)
.otherwise(wait_x))
sfn.StateMachine(
self,
"StateMachine",
definition=definition,
timeout=core.Duration.seconds(60),
)
def build(
scope: core.Construct,
id: str,
*,
default_fail_if_cluster_running: bool,
cluster_configuration_path: str = '$.ClusterConfiguration.Cluster',
output_path: str = '$',
result_path: str = '$.ClusterConfiguration.Cluster') -> sfn.Task:
# We use a nested Construct to avoid collisions with Lambda and Task ids
construct = core.Construct(scope, id)
fail_if_cluster_running_lambda = emr_lambdas.FailIfClusterRunningBuilder.get_or_build(
construct)
return sfn.Task(
construct,
'Fail If Cluster Running',
output_path=output_path,
result_path=result_path,
task=sfn_tasks.InvokeFunction(
fail_if_cluster_running_lambda,
payload={
'ExecutionInput':
sfn.TaskInput.from_context_at('$$.Execution.Input').value,
'DefaultFailIfClusterRunning':
default_fail_if_cluster_running,
'ClusterConfiguration':
sfn.TaskInput.from_data_at(
cluster_configuration_path).value
}))
def build(
scope: core.Construct,
id: str,
*,
cluster_configuration_path: str = '$.ClusterConfiguration.Cluster',
output_path: str = '$',
result_path: str = '$.ClusterConfiguration.Cluster') -> sfn.Task:
# We use a nested Construct to avoid collisions with Lambda and Task ids
construct = core.Construct(scope, id)
update_cluster_tags_lambda = emr_lambdas.UpdateClusterTagsBuilder.get_or_build(
construct)
return sfn.Task(
construct,
'Update Cluster Tags',
output_path=output_path,
result_path=result_path,
task=sfn_tasks.InvokeFunction(
update_cluster_tags_lambda,
payload={
'ExecutionInput':
sfn.TaskInput.from_context_at('$$.Execution.Input').value,
'ClusterConfiguration':
sfn.TaskInput.from_data_at(
cluster_configuration_path).value
}))
def build(
scope: core.Construct,
id: str,
*,
override_cluster_configs_lambda: Optional[
aws_lambda.Function] = None,
allowed_cluster_config_overrides: Optional[Dict[str, str]] = None,
cluster_configuration_path: str = '$.ClusterConfiguration.Cluster',
output_path: str = '$',
result_path: str = '$.ClusterConfiguration.Cluster') -> sfn.Task:
# We use a nested Construct to avoid collisions with Lambda and Task ids
construct = core.Construct(scope, id)
override_cluster_configs_lambda = \
emr_lambdas.OverrideClusterConfigsBuilder.get_or_build(construct) \
if override_cluster_configs_lambda is None \
else override_cluster_configs_lambda
return sfn.Task(
construct,
'Override Cluster Configs',
output_path=output_path,
result_path=result_path,
task=sfn_tasks.InvokeFunction(
override_cluster_configs_lambda,
payload={
'ExecutionInput':
sfn.TaskInput.from_context_at('$$.Execution.Input').value,
'ClusterConfiguration':
sfn.TaskInput.from_data_at(
cluster_configuration_path).value,
'AllowedClusterConfigOverrides':
allowed_cluster_config_overrides
}))
def _lambda_glue_crawler_task(self):
root_path = Path(os.path.dirname(os.path.abspath(__file__)))
lambda_handler = root_path.joinpath('lambdas',
'trigger_glue_crawler').as_posix()
func = lambda_.Function(
self,
"TriggerGlueCrawlerLambdaHandler",
handler="lambda.lambda_handler",
code=lambda_.AssetCode(lambda_handler),
environment={"crawlerName": f"{self.glue_crawler.name}"},
initial_policy=[
iam.PolicyStatement(
actions=["glue:StartCrawler"],
resources=["*"],
),
],
timeout=core.Duration.seconds(30),
runtime=lambda_.Runtime.PYTHON_3_7,
)
# turn the lambda into a stepfunction task so we can use it in our state machine
task = sfn.Task(
self,
"TriggerGlueCrawlerLambda",
task=sfnt.InvokeFunction(func),
)
return task
def __init__(self,
scope: core.Construct,
id: str,
*,
emr_step: emr_code.EMRStep,
cluster_id: str,
result_path: Optional[str] = None,
output_path: Optional[str] = None,
fail_chain: Optional[sfn.IChainable] = None,
wait_for_step_completion: bool = True):
super().__init__(scope, id)
override_step_args = emr_lambdas.OverrideStepArgsBuilder.get_or_build(
self)
override_step_args_task = sfn.Task(
self,
f'{emr_step.name} - Override Args',
result_path=f'$.{id}ResultArgs',
task=sfn_tasks.InvokeFunction(
override_step_args,
payload={
'ExecutionInput':
sfn.TaskInput.from_context_at('$$.Execution.Input').value,
'StepName':
emr_step.name,
'Args':
emr_step.args
}))
resolved_step = emr_step.resolve(self)
resolved_step['HadoopJarStep']['Args'] = sfn.TaskInput.from_data_at(
f'$.{id}ResultArgs').value
integration_pattern = sfn.ServiceIntegrationPattern.SYNC if wait_for_step_completion \
else sfn.ServiceIntegrationPattern.FIRE_AND_FORGET
add_step_task = sfn.Task(self,
emr_step.name,
output_path=output_path,
result_path=result_path,
task=emr_tasks.EmrAddStepTask(
cluster_id=cluster_id,
step=resolved_step,
integration_pattern=integration_pattern))
if fail_chain:
override_step_args_task.add_catch(fail_chain,
errors=['States.ALL'],
result_path='$.Error')
add_step_task.add_catch(fail_chain,
errors=['States.ALL'],
result_path='$.Error')
override_step_args_task.next(add_step_task)
self._start = override_step_args_task
self._end = add_step_task
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Step Function Starts Here
# The first thing we need to do is see if they are asking for pineapple on a pizza
pineapple_check_lambda = _lambda.Function(self, "pineappleCheckLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="orderPizza.handler",
code=_lambda.Code.from_asset("lambdas"),
)
# Step functions are built up of steps, we need to define our first step
order_pizza = step_fn.Task(self, 'Order Pizza Job',
task=step_fn_tasks.InvokeFunction(pineapple_check_lambda),
input_path='$.flavour',
result_path='$.pineappleAnalysis'
)
# Pizza Order failure step defined
job_failed = step_fn.Fail(self, 'Sorry, We Dont add Pineapple',
cause='Failed To Make Pizza',
error='They asked for Pineapple')
# If they didnt ask for pineapple let's cook the pizza
cook_pizza = step_fn.Pass(self, 'Lets make your pizza')
# If they ask for a pizza with pineapple, fail. Otherwise cook the pizza
definition = step_fn.Chain \
.start(order_pizza) \
.next(step_fn.Choice(self, 'With Pineapple?') \
.when(step_fn.Condition.boolean_equals('$.pineappleAnalysis.containsPineapple', True), job_failed) \
.otherwise(cook_pizza))
state_machine = step_fn.StateMachine(self, 'StateMachine', definition=definition, timeout=core.Duration.minutes(5))
# Dead Letter Queue Setup
dlq = sqs.Queue(self, 'stateMachineLambdaDLQ', visibility_timeout=core.Duration.seconds(300))
# defines an AWS Lambda resource to connect to our API Gateway
state_machine_lambda = _lambda.Function(self, "stateMachineLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="stateMachineLambda.handler",
code=_lambda.Code.from_asset("lambdas"),
environment={
'statemachine_arn': state_machine.state_machine_arn
}
)
state_machine.grant_start_execution(state_machine_lambda)
# defines an API Gateway REST API resource backed by our "sqs_publish_lambda" function.
api_gw.LambdaRestApi(self, 'Endpoint',
handler=state_machine_lambda
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
#lf1= Function(self, id="my_stack_lambda", runtime=Runtime.PYTHON_3_7, handler='handlers/my_lambda_handler', code='', function_name='my_example_lambda')
my_table = _dynamodb.Table(self,
id='dynamoTable',
table_name='testcdktabe',
partition_key=_dynamodb.Attribute(
name='lastname',
type=_dynamodb.AttributeType.STRING))
my_s3_bucket = _s3.Bucket(self,
id='s3bucket',
bucket_name='mynpbsample3bucket')
my_lambda_function = _lambda.Function(
self,
id='lambdafunction',
runtime=_lambda.Runtime.PYTHON_3_7,
handler='hello.handler',
code=_lambda.Code.asset('lambdacode'))
process_purchase_function = _lambda.Function(
self,
id='process_purchase',
runtime=_lambda.Runtime.PYTHON_3_7,
handler='process_purchase.handler',
code=_lambda.Code.asset('lambdacode'))
process_refund_function = _lambda.Function(
self,
id='process_refund',
runtime=_lambda.Runtime.PYTHON_3_7,
handler='process_refund.handler',
code=_lambda.Code.asset('lambdacode'))
#start_state = sfn.Pass(self, "start_state")
definition = sfn.Task(
self,
'Get Process Type',
task=tasks.InvokeFunction(process_purchase_function))
sfn.StateMachine(
self,
"MyStateMachine",
definition=definition,
timeout=core.Duration.seconds(30),
)
my_topic = sns.Topic(self,
"MyTopic",
display_name="Customer Subscription")
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
logging_lambda = lambda_func.Function(
scope=self,
id="logging_lambda",
function_name="logging_lambda",
handler="logging-lambda.main",
runtime=lambda_func.Runtime.PYTHON_3_7,
code=lambda_func.Code.from_asset("./code"))
second_lambda = lambda_func.Function(
scope=self,
id="second_lambda",
function_name="second_lambda",
handler="second-lambda.main",
runtime=lambda_func.Runtime.PYTHON_3_7,
code=lambda_func.Code.from_asset("./code"))
logging_lambda_task = tasks.InvokeFunction(logging_lambda)
logging_step = stepfunctions.Task(scope=self,
id="invoke_logging_function",
task=logging_lambda_task)
second_lambda_task = tasks.InvokeFunction(second_lambda)
second_step = stepfunctions.Task(scope=self,
id="invoke_second_function",
task=second_lambda_task)
definition = logging_step.next(second_step)
stepfunctions.StateMachine(
scope=self,
id="state_machine",
state_machine_name="state_machine",
definition=definition,
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# space for feeder Lambda function
feeder = aws_lambda.Function(self,
id='_feeder',
code=aws_lambda.Code.asset('./code'),
handler='feeder.handler',
runtime=aws_lambda.Runtime.PYTHON_3_7,
description='Feeder function for the Witness project')
# space for saver Lambda function
saver = aws_lambda.Function(self,
id='_saver',
code=aws_lambda.Code.asset('./code'),
handler='saver.handler',
runtime=aws_lambda.Runtime.PYTHON_3_7,
description='Saver function for the Witness project')
# space for feeder lambda trigger
archive.add_event_notification(aws_s3.EventType.OBJECT_CREATED_PUT, s3n.LambdaDestination(feeder))
# space for stepfunction
feederTask = aws_stepfunctions.Task( self,
id='_feederTask',
task=aws_tasks.InvokeFunction(feeder))
saverTask = aws_stepfunctions.Task( self,
id='_saverTask',
task=aws_tasks.InvokeFunction(saver))
definition = feederTask.next(saverTask)
orchestrator = aws_stepfunctions.StateMachine(self,
id='_orchestrator',
state_machine_name='witness_orchestrator',
definition=definition)
def build(scope: core.Construct,
id: str,
*,
cluster_name: str,
cluster_tags: List[core.Tag],
profile_namespace: str,
profile_name: str,
configuration_namespace: str,
configuration_name: str,
output_path: str = '$',
result_path: str = '$.ClusterConfiguration') -> sfn.Task:
# We use a nested Construct to avoid collisions with Lambda and Task ids
construct = core.Construct(scope, id)
load_cluster_configuration_lambda = emr_lambdas.LoadClusterConfigurationBuilder.build(
construct,
profile_namespace=profile_namespace,
profile_name=profile_name,
configuration_namespace=configuration_namespace,
configuration_name=configuration_name)
return sfn.Task(construct,
'Load Cluster Configuration',
output_path=output_path,
result_path=result_path,
task=sfn_tasks.InvokeFunction(
load_cluster_configuration_lambda,
payload={
'ClusterName':
cluster_name,
'ClusterTags': [{
'Key': t.key,
'Value': t.value
} for t in cluster_tags],
'ProfileNamespace':
profile_namespace,
'ProfileName':
profile_name,
'ConfigurationNamespace':
configuration_namespace,
'ConfigurationName':
configuration_name,
}))
def __init__(self,
scope: core.Construct,
id: str,
name: str,
state_machine: sfn.StateMachine,
input: Optional[Mapping[str, any]] = None,
fail_chain: Optional[sfn.IChainable] = None):
super().__init__(scope, id)
state_machine_task = sfn.Task(
self,
name,
task=emr_tasks.StartExecutionTask(
state_machine=state_machine,
input=input,
integration_pattern=sfn.ServiceIntegrationPattern.SYNC))
parse_json_string = emr_lambdas.ParseJsonStringBuilder.get_or_build(
self)
parse_json_string_task = sfn.Task(
self,
f'{name} - Parse JSON Output',
result_path='$',
task=sfn_tasks.InvokeFunction(
parse_json_string,
payload={
'JsonString': sfn.TaskInput.from_data_at('$.Output').value
}))
if fail_chain:
state_machine_task.add_catch(fail_chain,
errors=['States.ALL'],
result_path='$.Error')
parse_json_string_task.add_catch(fail_chain,
errors=['States.ALL'],
result_path='$.Error')
state_machine_task.next(parse_json_string_task)
self._start = state_machine_task
self._end = parse_json_string_task
def _lambda_quality_check_task(self):
lambda_role = iam.Role(
self,
id=f"QualityLambdaRole",
assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonS3FullAccess"),
iam.ManagedPolicy.from_aws_managed_policy_name(
"AmazonAthenaFullAccess"),
],
)
root_path = Path(os.path.dirname(os.path.abspath(__file__)))
lambda_handler = root_path.joinpath('lambdas',
'quality_check').as_posix()
func = lambda_.Function(
self,
"QualityCheckAthenaLambdaHandler",
handler="lambda.lambda_handler",
code=lambda_.AssetCode(lambda_handler),
environment={"athenaDatabase": f"{self.glue_db_name}"},
role=lambda_role,
timeout=core.Duration.seconds(30),
runtime=lambda_.Runtime.PYTHON_3_7,
)
# turn the lambda into a stepfunction task so we can use it in our state machine
task = sfn.Task(
self,
"QualityCheckAthenaLambda",
task=sfnt.InvokeFunction(func),
)
return task
def __init__(self,
scope: core.Construct,
id: str,
id_checker: str,
event_bus: str,
stage: Optional[str] = 'prod',
**kwargs) -> None:
super().__init__(scope, id + '-' + stage, **kwargs)
app_table_name = id + '-applications-table-' + stage
app_table = ddb.Table(self,
id=app_table_name,
table_name=app_table_name,
partition_key=ddb.Attribute(
name='id', type=ddb.AttributeType.STRING),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST)
events_table_name = id + '-events-table-' + stage
events_table = ddb.Table(self,
id=events_table_name,
table_name=events_table_name,
partition_key=ddb.Attribute(
name='id', type=ddb.AttributeType.STRING),
billing_mode=ddb.BillingMode.PAY_PER_REQUEST,
stream=ddb.StreamViewType.NEW_IMAGE)
self._table_stream_arn = events_table.table_stream_arn
# create our Lambda function for the bank account service
func_name = id + '-' + stage + '-' + 'account-application'
lambda_assets = lambda_.Code.from_asset('account_application_service')
handler = lambda_.Function(self,
func_name,
code=lambda_assets,
runtime=lambda_.Runtime.NODEJS_10_X,
handler='main.handler',
environment={
'ACCOUNTS_TABLE_NAME':
app_table.table_name,
'EVENTS_TABLE_NAME':
events_table.table_name,
'REGION': core.Aws.REGION
})
gw.LambdaRestApi(self, id=stage + '-' + id, handler=handler)
# grant main Lambda function access to DynamoDB tables
app_table.grant_read_write_data(handler.role)
events_table.grant_read_write_data(handler.role)
p_statement = iam.PolicyStatement(actions=[
'ssm:Describe*', 'ssm:Get*', 'ssm:List*', 'events:*', 'states:*'
],
effect=iam.Effect.ALLOW,
resources=['*'])
handler.add_to_role_policy(statement=p_statement)
# create the Lambda function for the event publisher
evt_publisher = id + '-' + stage + '-' + 'event-publisher'
evt_handler = lambda_.Function(
self,
evt_publisher,
code=lambda_assets,
runtime=lambda_.Runtime.NODEJS_10_X,
handler='event-publisher.handler',
events=[
lambda_es.DynamoEventSource(
table=events_table,
starting_position=lambda_.StartingPosition.LATEST)
],
environment={
'EVENT_BRIDGE_ARN': event_bus,
'REGION': core.Aws.REGION
})
evt_handler.add_to_role_policy(statement=p_statement)
# set up StepFunctions
approve_application = sf.Task(
self,
'Approve Application',
task=sft.InvokeFunction(handler,
payload={
'body': {
'command':
'APPROVE_ACCOUNT_APPLICATION',
'data': {
'id.$': '$.application.id'
}
}
}),
result_path='$.approveApplication')
reject_application = sf.Task(self,
'Reject Application',
task=sft.InvokeFunction(
handler,
payload={
'body': {
'command':
'REJECT_ACCOUNT_APPLICATION',
'data': {
'id.$': '$.application.id'
}
}
}),
result_path='$.rejectApplication')
id_checker_handler = lambda_.Function.from_function_arn(
self, 'IdentityChecker', function_arn=id_checker)
check_identity = sf.Task(self,
'Check Identity',
task=sft.InvokeFunction(
id_checker_handler,
payload={
'body': {
'command': 'CHECK_IDENTITY',
'data': {
'application.$':
'$.application'
}
}
}))
wait_for_human_review = sf.Task(self, 'Wait for Human Review',
task=sft.RunLambdaTask(handler,
integration_pattern=sf.ServiceIntegrationPattern.WAIT_FOR_TASK_TOKEN,
payload={
'body': {
'command': 'FLAG_ACCOUNT_APPLICATION_FOR_HUMAN_REVIEW',
'data': {
'id.$': '$.application.id',
'taskToken': sf.Context.task_token
}
}
}), result_path='$.humanReview') \
.next(
sf.Choice(self, 'Human Approval Choice')
.when(sf.Condition.string_equals('$.humanReview.decision', 'APPROVE'), next=approve_application)
.when(sf.Condition.string_equals('$.humanReview.decision', 'REJECT'), next=reject_application))
sm_definition = sf.Parallel(self, 'Perform Automated Checks', result_path='$.checks') \
.branch(check_identity) \
.branch(sf.Pass(self, 'Check Fraud Model', result=sf.Result({'flagged': False}))) \
.next(
sf.Choice(self, 'Automated Checks Choice')
.when(sf.Condition.boolean_equals('$.checks[0].flagged', True), next=wait_for_human_review)
.when(sf.Condition.boolean_equals('$.checks[1].flagged', True), next=wait_for_human_review)
.otherwise(approve_application))
state_machine = sf.StateMachine(self,
'OpenAccountStateMachine' + stage,
definition=sm_definition)
ssm.CfnParameter(self,
id='StateMachineArnSSM',
type='String',
value=state_machine.state_machine_arn,
name='StateMachineArnSSM')
def __init__(self, app: core.App, id: str, props, **kwargs) -> None:
super().__init__(app, id, **kwargs)
run_data_bucket_name = ''
run_data_bucket = s3.Bucket.from_bucket_name(
self, run_data_bucket_name, bucket_name=run_data_bucket_name)
# IAM roles for the lambda functions
lambda_role = iam.Role(
self,
'EchoTesLambdaRole',
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole')
])
copy_lambda_role = iam.Role(
self,
'CopyToS3LambdaRole',
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole')
])
run_data_bucket.grant_write(copy_lambda_role)
callback_role = iam.Role(
self,
'CallbackTesLambdaRole',
assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'),
managed_policies=[
iam.ManagedPolicy.from_aws_managed_policy_name(
'service-role/AWSLambdaBasicExecutionRole'),
iam.ManagedPolicy.from_aws_managed_policy_name(
'AWSStepFunctionsFullAccess')
])
# Lambda function to call back and complete SFN async tasks
lmbda.Function(self,
'CallbackLambda',
function_name='callback_iap_tes_lambda_dev',
handler='callback.lambda_handler',
runtime=lmbda.Runtime.PYTHON_3_7,
code=lmbda.Code.from_asset('lambdas'),
role=callback_role,
timeout=core.Duration.seconds(20))
samplesheet_mapper_function = lmbda.Function(
self,
'SampleSheetMapperTesLambda',
function_name='showcase_ss_mapper_iap_tes_lambda_dev',
handler='launch_tes_task.lambda_handler',
runtime=lmbda.Runtime.PYTHON_3_7,
code=lmbda.Code.from_asset('lambdas'),
role=lambda_role,
timeout=core.Duration.seconds(20),
environment={
'IAP_API_BASE_URL': props['iap_api_base_url'],
'TASK_ID': props['task_id'],
'TASK_VERSION': 'tvn.0ee81865bf514b7bb7b7ea305c88191f',
# 'TASK_VERSION': 'tvn.b4735419fbe4455eb2b91960e48921f9', # echo task
'SSM_PARAM_JWT': props['ssm_param_name'],
'GDS_LOG_FOLDER': props['gds_log_folder'],
'IMAGE_NAME': 'umccr/alpine_pandas',
'IMAGE_TAG': '1.0.1',
'TES_TASK_NAME': 'SampleSheetMapper'
})
bcl_convert_function = lmbda.Function(
self,
'BclConvertTesLambda',
function_name='showcase_bcl_convert_iap_tes_lambda_dev',
handler='launch_tes_task.lambda_handler',
runtime=lmbda.Runtime.PYTHON_3_7,
code=lmbda.Code.from_asset('lambdas'),
role=lambda_role,
timeout=core.Duration.seconds(20),
environment={
'IAP_API_BASE_URL': props['iap_api_base_url'],
'TASK_ID': props['task_id'],
'TASK_VERSION': 'tvn.ab3e85f9aaf24890ad169fdab3825c0d',
# 'TASK_VERSION': 'tvn.b4735419fbe4455eb2b91960e48921f9', # echo task
'SSM_PARAM_JWT': props['ssm_param_name'],
'GDS_LOG_FOLDER': props['gds_log_folder'],
'IMAGE_NAME':
'699120554104.dkr.ecr.us-east-1.amazonaws.com/public/dragen',
'IMAGE_TAG': '3.5.2',
'TES_TASK_NAME': 'BclConvert'
})
fastq_mapper_function = lmbda.Function(
self,
'FastqMapperTesLambda',
function_name='showcase_fastq_mapper_iap_tes_lambda_dev',
handler='launch_tes_task.lambda_handler',
runtime=lmbda.Runtime.PYTHON_3_7,
code=lmbda.Code.from_asset('lambdas'),
role=lambda_role,
timeout=core.Duration.seconds(20),
environment={
'IAP_API_BASE_URL': props['iap_api_base_url'],
'TASK_ID': props['task_id'],
'TASK_VERSION': 'tvn.f90aa88da2fe490fb6e6366b65abe267',
# 'TASK_VERSION': 'tvn.b4735419fbe4455eb2b91960e48921f9', # echo task
'SSM_PARAM_JWT': props['ssm_param_name'],
'GDS_LOG_FOLDER': props['gds_log_folder'],
'IMAGE_NAME': 'umccr/alpine_pandas',
'IMAGE_TAG': '1.0.1',
'TES_TASK_NAME': 'FastqMapper'
})
gather_samples_function = lmbda.Function(
self,
'GatherSamplesTesLambda',
function_name='showcase_gather_samples_iap_tes_lambda_dev',
handler='gather_samples.lambda_handler',
runtime=lmbda.Runtime.PYTHON_3_7,
code=lmbda.Code.from_asset('lambdas'),
role=lambda_role,
timeout=core.Duration.seconds(20),
environment={
'IAP_API_BASE_URL': props['iap_api_base_url'],
'SSM_PARAM_JWT': props['ssm_param_name']
})
dragen_function = lmbda.Function(
self,
'DragenTesLambda',
function_name='showcase_dragen_iap_tes_lambda_dev',
handler='launch_tes_task.lambda_handler',
runtime=lmbda.Runtime.PYTHON_3_7,
code=lmbda.Code.from_asset('lambdas'),
role=lambda_role,
timeout=core.Duration.seconds(20),
environment={
'IAP_API_BASE_URL': props['iap_api_base_url'],
'TASK_ID': props['task_id'],
'TASK_VERSION': 'tvn.096b39e90e4443abae0333e23fcabc61',
# 'TASK_VERSION': 'tvn.b4735419fbe4455eb2b91960e48921f9', # echo task
'SSM_PARAM_JWT': props['ssm_param_name'],
'GDS_LOG_FOLDER': props['gds_log_folder'],
'IMAGE_NAME':
'699120554104.dkr.ecr.us-east-1.amazonaws.com/public/dragen',
'IMAGE_TAG': '3.5.2',
'TES_TASK_NAME': 'Dragen'
})
multiqc_function = lmbda.Function(
self,
'MultiQcTesLambda',
function_name='showcase_multiqc_iap_tes_lambda_dev',
handler='launch_tes_task.lambda_handler',
runtime=lmbda.Runtime.PYTHON_3_7,
code=lmbda.Code.from_asset('lambdas'),
role=lambda_role,
timeout=core.Duration.seconds(20),
environment={
'IAP_API_BASE_URL': props['iap_api_base_url'],
'TASK_ID': props['task_id'],
'TASK_VERSION': 'tvn.983a0239483d4253a8a0531fa1de0376',
# 'TASK_VERSION': 'tvn.b4735419fbe4455eb2b91960e48921f9', # echo task
'SSM_PARAM_JWT': props['ssm_param_name'],
'GDS_LOG_FOLDER': props['gds_log_folder'],
'IMAGE_NAME': 'umccr/multiqc_dragen',
'IMAGE_TAG': '1.1',
'TES_TASK_NAME': 'MultiQC'
})
copy_report_to_s3 = lmbda.Function(
self,
'CopyReportToS3Lambda',
function_name='showcase_copy_report_lambda_dev',
handler='copy_to_s3.lambda_handler',
runtime=lmbda.Runtime.PYTHON_3_7,
code=lmbda.Code.from_asset('lambdas'),
role=copy_lambda_role,
timeout=core.Duration.seconds(20),
environment={
'IAP_API_BASE_URL': props['iap_api_base_url'],
'SSM_PARAM_JWT': props['ssm_param_name'],
'GDS_RUN_VOLUME': props['gds_run_volume'],
'S3_RUN_BUCKET': props['s3_run_bucket']
})
# IAP JWT access token stored in SSM Parameter Store
secret_value = ssm.StringParameter.from_secure_string_parameter_attributes(
self,
"JwtToken",
parameter_name=props['ssm_param_name'],
version=props['ssm_param_version'])
secret_value.grant_read(samplesheet_mapper_function)
secret_value.grant_read(bcl_convert_function)
secret_value.grant_read(fastq_mapper_function)
secret_value.grant_read(gather_samples_function)
secret_value.grant_read(dragen_function)
secret_value.grant_read(multiqc_function)
secret_value.grant_read(copy_report_to_s3)
# SFN task definitions
task_samplesheet_mapper = sfn.Task(
self,
"SampleSheetMapper",
task=sfn_tasks.RunLambdaTask(
samplesheet_mapper_function,
integration_pattern=sfn.ServiceIntegrationPattern.
WAIT_FOR_TASK_TOKEN,
payload={
"taskCallbackToken": sfn.Context.task_token,
"runId.$": "$.runfolder"
}),
result_path="$.guid")
task_bcl_convert = sfn.Task(
self,
"BclConvert",
task=sfn_tasks.RunLambdaTask(
bcl_convert_function,
integration_pattern=sfn.ServiceIntegrationPattern.
WAIT_FOR_TASK_TOKEN,
payload={
"taskCallbackToken": sfn.Context.task_token,
"runId.$": "$.runfolder"
}),
result_path="$.guid")
task_fastq_mapper = sfn.Task(
self,
"FastqMapper",
task=sfn_tasks.RunLambdaTask(
fastq_mapper_function,
integration_pattern=sfn.ServiceIntegrationPattern.
WAIT_FOR_TASK_TOKEN,
payload={
"taskCallbackToken": sfn.Context.task_token,
"runId.$": "$.runfolder"
}),
result_path="$.guid")
task_gather_samples = sfn.Task(self,
"GatherSamples",
task=sfn_tasks.InvokeFunction(
gather_samples_function,
payload={"runId.$": "$.runfolder"}),
result_path="$.sample_ids")
task_dragen = sfn.Task(
self,
"DragenTask",
task=sfn_tasks.RunLambdaTask(
dragen_function,
integration_pattern=sfn.ServiceIntegrationPattern.
WAIT_FOR_TASK_TOKEN,
payload={
"taskCallbackToken": sfn.Context.task_token,
"runId.$": "$.runId",
"index.$": "$.index",
"item.$": "$.item"
}),
result_path="$.exit_status")
task_multiqc = sfn.Task(
self,
"MultiQcTask",
task=sfn_tasks.RunLambdaTask(
multiqc_function,
integration_pattern=sfn.ServiceIntegrationPattern.
WAIT_FOR_TASK_TOKEN,
payload={
"taskCallbackToken": sfn.Context.task_token,
"runId.$": "$.runfolder",
"samples.$": "$.sample_ids"
}))
task_copy_report_to_s3 = sfn.Task(
self,
"CopyReportToS3",
task=sfn_tasks.InvokeFunction(copy_report_to_s3,
payload={"runId.$": "$.runfolder"}),
result_path="$.copy_report")
scatter = sfn.Map(self,
"Scatter",
items_path="$.sample_ids",
parameters={
"index.$": "$$.Map.Item.Index",
"item.$": "$$.Map.Item.Value",
"runId.$": "$.runfolder"
},
result_path="$.mapresults",
max_concurrency=20).iterator(task_dragen)
definition = task_samplesheet_mapper \
.next(task_bcl_convert) \
.next(task_fastq_mapper) \
.next(task_gather_samples) \
.next(scatter) \
.next(task_multiqc) \
.next(task_copy_report_to_s3)
sfn.StateMachine(
self,
"ShowcaseSfnStateMachine",
definition=definition,
)
def __init__(self, app: core.App, id: str, **kwargs) -> None:
super().__init__(app, id, **kwargs)
# Create both lambdas
with open("lambda-submit.py", encoding="utf8") as fp:
lambda_submit_code = fp.read()
lambdaFn1 = lambda_.Function(
self,
"submitsmbatch",
code=lambda_.InlineCode(lambda_submit_code),
handler="index.lambda_handler",
timeout=core.Duration.seconds(300),
runtime=lambda_.Runtime.PYTHON_3_7,
environment={
"transform_job_name":transform_job_name,
"model_name":model_name,
"max_concurrent":max_concurrent,
"max_payload_size":max_payload_size,
"s3_uri_in":s3_uri_in,
"s3_uri_out":s3_uri_out,
"instance_type":instance_type,
"instance_count":instance_count,
}
)
# Add perms
lambdaFn1.add_to_role_policy(aws_iam.PolicyStatement(
actions = ['sagemaker:CreateTransformJob',],
resources = ['arn:aws:sagemaker:{}:{}:transform-job/{}*'.format(my_region,my_acc_id,transform_job_name),]
))
with open("lambda-check.py", encoding="utf8") as fp:
lambda_check_code = fp.read()
lambdaFn2 = lambda_.Function(
self,
"checksmbatch",
code=lambda_.InlineCode(lambda_check_code),
handler="index.lambda_handler",
timeout=core.Duration.seconds(300),
runtime=lambda_.Runtime.PYTHON_3_7,
environment={"model_name":model_name, # CHANGE TO YOUR ENDPOINT NAME!!
"content_type":"text/csv"}
)
# Add perms
lambdaFn2.add_to_role_policy(aws_iam.PolicyStatement(
actions = ['sagemaker:DescribeTransformJob',],
resources = ['arn:aws:sagemaker:{}:{}:transform-job/{}*'.format(my_region,my_acc_id,transform_job_name),]
))
# Define state machine
# submit_job_activity = sfn.Activity(
# self, "SubmitJob"
# )
# check_job_activity = sfn.Activity(
# self, "CheckJob"
# )
submit_job = sfn.Task(
self, "Submit Job",
task=sfn_tasks.InvokeFunction(lambdaFn1),
)
wait_x = sfn.Wait(
self, "Wait 1 minute",
time=sfn.WaitTime.duration(core.Duration.minutes(1)),
)
get_status = sfn.Task(
self, "Get Job Status",
task=sfn_tasks.InvokeFunction(lambdaFn2),
)
is_complete = sfn.Choice(
self, "Job Complete?"
)
job_failed = sfn.Fail(
self, "Job Failed",
cause="AWS Batch Job Failed",
error="DescribeJob returned FAILED"
)
final_status = sfn.Task(
self, "Get Final Job Status",
task=sfn_tasks.InvokeFunction(lambdaFn2),
)
definition = submit_job\
.next(wait_x)\
.next(get_status)\
.next(is_complete
.when(sfn.Condition.string_equals(
"$.status", "Failed"), job_failed)
.when(sfn.Condition.string_equals(
"$.status", "Completed"), final_status)
.otherwise(wait_x))
sfn.StateMachine(
self, "SMbatchInference",
definition=definition,
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# Lets create couple of instances to test):
vpc = _ec2.Vpc(self,
"abacVPC",
cidr="10.13.0.0/21",
max_azs=2,
nat_gateways=0,
subnet_configuration=[
_ec2.SubnetConfiguration(
name="pubSubnet",
cidr_mask=24,
subnet_type=_ec2.SubnetType.PUBLIC)
])
core.Tag.add(vpc,
key="ServiceProvider",
value="KonStone",
include_resource_types=[])
weak_sg = _ec2.SecurityGroup(
self,
"web_sec_grp",
vpc=vpc,
description="Allow internet access from the world",
allow_all_outbound=True)
# vpc_cidr_block
# weak_sg.add_ingress_rule(_ec2.Peer.any_ipv4(),
weak_sg.add_ingress_rule(_ec2.Peer.ipv4(vpc.vpc_cidr_block),
_ec2.Port.tcp(22),
"Allow SSH access from the VPC Only.")
# We are using the latest AMAZON LINUX AMI
# Benefit of having SSM Agent pre-installed
ami_id = _ec2.AmazonLinuxImage(generation=_ec2.AmazonLinuxGeneration.
AMAZON_LINUX_2).get_image(self).image_id
# https://docs.aws.amazon.com/cdk/api/latest/python/aws_cdk.aws_iam/Role.html
instace_profile_role = _iam.Role(
self,
'ec2ssmroleid',
assumed_by=_iam.ServicePrincipal('ec2.amazonaws.com'),
role_name="instace_profile_role")
instace_profile_role.add_managed_policy(
_iam.ManagedPolicy.from_aws_managed_policy_name(
'AmazonSSMManagedInstanceCore'))
instance_profile_role_additional_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
"arn:aws:logs:*:*:*",
],
actions=["logs:Create*", "logs:PutLogEvents"])
instance_profile_role_additional_perms.sid = "PutBucketPolicy"
instace_profile_role.add_to_policy(
instance_profile_role_additional_perms)
inst_profile_01 = _iam.CfnInstanceProfile(
self,
"instProfile01Id",
roles=[instace_profile_role.role_name],
)
# Let us bootstrap the server with the required agents
try:
with open("./bootstrap_scripts/install_agents.sh",
mode='rb') as file:
bootstrap_data = file.read()
except OSError:
print('Failed to get UserData script')
install_agents = _ec2.UserData.for_linux()
install_agents.add_commands(str(bootstrap_data, 'utf-8'))
# The EC2 Instance to monitor for failed SSH Logins
ssh_monitored_inst_01 = _ec2.CfnInstance(
self,
"sshMonitoredInstance01",
image_id=ami_id,
instance_type="t2.micro",
monitoring=False,
tags=[{
"key": "ServiceProvider",
"value": "KonStone"
}],
iam_instance_profile=inst_profile_01.ref,
network_interfaces=[{
"deviceIndex": "0",
"associatePublicIpAddress": True,
"subnetId": vpc.public_subnets[0].subnet_id,
"groupSet": [weak_sg.security_group_id]
}], #https: //github.com/aws/aws-cdk/issues/3419
user_data=core.Fn.base64(install_agents.render()),
)
"""
linux_ami = _ec2.GenericLinuxImage({ "cn-northwest-1": "ami-0f62e91915e16cfc2","eu-west-1": "ami-12345678"})
ssh_monitored_inst_01_02 = _ec2.Instance(self,
"monitoredInstance02",
instance_type=_ec2.InstanceType(instance_type_identifier="t2.micro"),
instance_name="monitoredInstance02",
machine_image=linux_ami,
vpc=vpc,
security_group=[weak_sg.security_group_id],
# vpc_subnets=_ec2.SubnetSelection(subnet_type=_ec2.SubnetType.PUBLIC)
vpc_subnets=vpc.public_subnets[0].subnet_id,
# user_data=_ec2.UserData.custom(t_user_data)
)
"""
# The log group name to store logs
info_sec_ops_log_group = _logs.LogGroup(
self,
"infoSecOpsLogGroupId",
log_group_name=(f"/Mystique/InfoSec/Automation/"
f"{ssh_monitored_inst_01.ref}"),
retention=_logs.RetentionDays.ONE_WEEK)
# Defines an AWS Lambda resource
with open("lambda_src/quarantine_ec2_instance.py",
encoding="utf8") as fp:
quarantine_ec2_instance_fn_handler_code = fp.read()
quarantine_ec2_instance_fn = _lambda.Function(
self,
id='quarantineEc2InstanceFnId',
function_name="quarantine_ec2_instance",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(quarantine_ec2_instance_fn_handler_code),
handler='index.lambda_handler',
timeout=core.Duration.seconds(5))
quarantine_ec2_instance_fn_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
"*",
],
actions=[
"ec2:RevokeSecurityGroupIngress",
"ec2:DescribeSecurityGroupReferences",
"ec2:RevokeSecurityGroupEgress",
"ec2:ApplySecurityGroupsToClientVpnTargetNetwork",
"ec2:DescribeSecurityGroups", "ec2:CreateSecurityGroup",
"ec2:DescribeInstances", "ec2:CreateTags", "ec2:StopInstances",
"ec2:CreateVolume", "ec2:CreateSnapshots",
"ec2:CreateSnapshot", "ec2:DescribeSnapshots",
"ec2:ModifyInstanceAttribute"
])
quarantine_ec2_instance_fn_perms.sid = "AllowLambdaToQuarantineEC2"
quarantine_ec2_instance_fn.add_to_role_policy(
quarantine_ec2_instance_fn_perms)
info_sec_ops_topic = _sns.Topic(self,
"infoSecOpsTopicId",
display_name="InfoSecTopic",
topic_name="InfoSecOpsTopic")
# Ref: https://docs.aws.amazon.com/cdk/api/latest/docs/aws-stepfunctions-readme.html
###############################################################################
################# STEP FUNCTIONS EXPERIMENTAL CODE - UNSTABLE #################
###############################################################################
quarantine_ec2_instance_task = _sfn.Task(
self,
"Quarantine EC2 Instance",
task=_tasks.InvokeFunction(quarantine_ec2_instance_fn),
result_path="$")
notify_secops_task = _sfn.Task(
self,
"Notify InfoSecOps",
task=_tasks.PublishToTopic(
info_sec_ops_topic,
integration_pattern=_sfn.ServiceIntegrationPattern.
FIRE_AND_FORGET,
message=_sfn.TaskInput.from_data_at("$.message"),
subject="SSH Error Response Notification"))
ssh_error_response_failure = _sfn.Fail(
self,
"SSH Error Response Actions Failed",
cause="All Response Actions were NOT completed",
error="Check Logs")
ssh_error_response_success = _sfn.Succeed(
self,
"SSH Error Response Actions Succeeded",
comment="All Response Action Completed Successfully",
)
ssh_error_response_sfn_definition = quarantine_ec2_instance_task\
.next(notify_secops_task\
.next(_sfn.Choice(self, "SSH Errors Response Complete?")\
.when(_sfn.Condition.number_equals("$.SdkHttpMetadata.HttpStatusCode", 200),ssh_error_response_success)\
.when(_sfn.Condition.not_(
_sfn.Condition.number_equals("$.SdkHttpMetadata.HttpStatusCode", 200)), ssh_error_response_failure)\
.otherwise(ssh_error_response_failure)
)
)
ssh_error_response_statemachine = _sfn.StateMachine(
self,
"stateMachineId",
definition=ssh_error_response_sfn_definition,
timeout=core.Duration.minutes(5))
###############################################################################
################# STEP FUNCTIONS EXPERIMENTAL CODE - UNSTABLE #################
###############################################################################
# LAMBDA TO TRIGGER STATE MACHINE - since state cannot be invoked by SNS
with open("lambda_src/trigger_state_machine.py",
encoding="utf8") as fp:
trigger_state_machine_fn_handler_code = fp.read()
trigger_state_machine_fn = _lambda.Function(
self,
id='sshErrorResponseFnId',
function_name="trigger_ssh_error_response_state_machine_fn",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(trigger_state_machine_fn_handler_code),
# code=_lambda.Code.asset("lambda_src/is_policy_permissive.py"),
# code=_lambda.Code.asset('lambda_src'),
# code=_lambda.InlineCode(code_body),
handler='index.lambda_handler',
timeout=core.Duration.seconds(5),
environment={
"STATE_MACHINE_ARN":
f"{ssh_error_response_statemachine.state_machine_arn}",
})
trigger_state_machine_fn_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
f"{ssh_error_response_statemachine.state_machine_arn}",
],
actions=["states:StartExecution"])
trigger_state_machine_fn_perms.sid = "PutBucketPolicy"
trigger_state_machine_fn.add_to_role_policy(
trigger_state_machine_fn_perms)
"""
version = trigger_state_machine_fn.add_version(name=datetime.now().isoformat())
trigger_state_machine_fn_alias = _lambda.Alias(self,
'lmdaAliasId',
alias_name='MystiqueTestAlias',
version=version
)
"""
# Lets add permission to SNS to trigger our lambda function
trigger_lambda_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
trigger_state_machine_fn.function_arn,
],
actions=[
"lambda:InvokeFunction",
])
trigger_lambda_perms.sid = "TriggerLambaFunction"
# info_sec_ops_topic.add_to_resource_policy( trigger_lambda_perms )
# Subscribe InfoSecOps Email to topic
info_sec_ops_topic.add_subscription(
_subs.EmailSubscription(global_args.INFO_SEC_OPS_EMAIL))
# info_sec_ops_topic.add_subscription(_subs.LambdaSubscription(trigger_state_machine_fn))
trigger_state_machine_fn_alarm = trigger_state_machine_fn.metric_all_errors(
).create_alarm(
self,
"fn-error-alarm",
threshold=5,
alarm_name="trigger_state_machine_fn_error_alarm",
evaluation_periods=5,
period=core.Duration.minutes(1),
)
subscribe_trigger_state_machine_fn_to_logs = _logs.SubscriptionFilter(
self,
"sshErrorLogSubscriptionId",
log_group=info_sec_ops_log_group,
destination=_logs_destination.LambdaDestination(
trigger_state_machine_fn),
filter_pattern=_logs.FilterPattern.space_delimited(
"Mon", "day", "timestamp", "ip", "id", "status",
"...").where_string("status", "=", "Invalid"),
)
# https://pypi.org/project/aws-cdk.aws-logs/
# We are creating three filter
# tooManySshDisconnects, invalidSshUser and invalidSshKey:
# When a user tries to SSH with invalid username the next line is logged in the SSH log file:
# Apr 20 02:39:35 ip-172-31-63-56 sshd[17136]: Received disconnect from xxx.xxx.xxx.xxx: 11: [preauth]
too_many_ssh_disconnects_metric = _cloudwatch.Metric(
namespace=f"{global_args.OWNER}",
metric_name="tooManySshDisconnects")
too_many_ssh_disconnects_filter = _logs.MetricFilter(
self,
"tooManySshDisconnectsFilterId",
log_group=info_sec_ops_log_group,
metric_namespace=too_many_ssh_disconnects_metric.namespace,
metric_name=too_many_ssh_disconnects_metric.metric_name,
filter_pattern=_logs.FilterPattern.space_delimited(
"Mon", "day", "timestamp", "ip", "id", "msg1", "msg2",
"...").where_string("msg2", "=", "disconnect"),
metric_value="1")
invalid_ssh_user_metric = _cloudwatch.Metric(
namespace=f"{global_args.OWNER}",
metric_name="invalidSshUser",
)
invalid_ssh_user_filter = _logs.MetricFilter(
self,
"invalidSshUserFilterId",
log_group=info_sec_ops_log_group,
metric_namespace=invalid_ssh_user_metric.namespace,
metric_name=invalid_ssh_user_metric.metric_name,
filter_pattern=_logs.FilterPattern.space_delimited(
"Mon", "day", "timestamp", "ip", "id", "status",
"...").where_string("status", "=", "Invalid"),
metric_value="1")
invalid_ssh_key_metric = _cloudwatch.Metric(
namespace=f"{global_args.OWNER}", metric_name="invalidSshKey")
invalid_ssh_key_filter = _logs.MetricFilter(
self,
"invalidSshKeyFilterId",
log_group=info_sec_ops_log_group,
metric_namespace=invalid_ssh_key_metric.namespace,
metric_name=invalid_ssh_key_metric.metric_name,
filter_pattern=_logs.FilterPattern.space_delimited(
"Mon", "day", "timestamp", "ip", "id", "msg1", "msg2",
"...").where_string("msg1", "=", "Connection").where_string(
"msg2", "=", "closed"),
metric_value="1")
# Now let us create alarms
# alarm is raised there are more than 5(threshold) of the measured metrics in two(datapoint) of the last three seconds(evaluation):
# Period=60Seconds, Eval=3, Threshold=5
too_many_ssh_disconnects_alarm = _cloudwatch.Alarm(
self,
"tooManySshDisconnectsAlarmId",
alarm_name="too_many_ssh_disconnects_alarm",
alarm_description=
"The number disconnect requests is greater then 5, even 1 time in 3 minutes",
metric=too_many_ssh_disconnects_metric,
actions_enabled=True,
period=core.Duration.minutes(1),
threshold=5,
evaluation_periods=3,
datapoints_to_alarm=1,
statistic="sum",
comparison_operator=_cloudwatch.ComparisonOperator.
GREATER_THAN_OR_EQUAL_TO_THRESHOLD)
invalid_ssh_user_alarm = _cloudwatch.Alarm(
self,
"invalidSshUserAlarmId",
alarm_name="too_many_invalid_ssh_users_alarm",
alarm_description=
"The number of invalid ssh users connecting is greater then 5, even 1 time in 3 minutes",
metric=invalid_ssh_user_metric,
actions_enabled=True,
period=core.Duration.minutes(1),
threshold=5,
evaluation_periods=3,
datapoints_to_alarm=1,
statistic="sum",
comparison_operator=_cloudwatch.ComparisonOperator.
GREATER_THAN_THRESHOLD)
invalid_ssh_user_alarm.add_alarm_action(
_cloudwatch_actions.SnsAction(info_sec_ops_topic))
invalid_ssh_key_alarm = _cloudwatch.Alarm(
self,
"invalidSshKeyAlarmId",
alarm_name="too_many_invalid_ssh_key_alarm",
alarm_description=
"The number of invalid ssh keys connecting is greater then 5, even 1 time in 3 minutes",
metric=invalid_ssh_key_metric,
actions_enabled=True,
period=core.Duration.minutes(1),
threshold=5,
evaluation_periods=3,
datapoints_to_alarm=1,
statistic="sum",
comparison_operator=_cloudwatch.ComparisonOperator.
GREATER_THAN_OR_EQUAL_TO_THRESHOLD)
invalid_ssh_key_alarm.add_alarm_action(
_cloudwatch_actions.SnsAction(info_sec_ops_topic))
###########################################
################# OUTPUTS #################
###########################################
output0 = core.CfnOutput(
self,
"SecuirtyAutomationFrom",
value=f"{global_args.SOURCE_INFO}",
description=
"To know more about this automation stack, check out our github page."
)
output1_1 = core.Fn.get_att(
logical_name_of_resource="sshMonitoredInstance01",
attribute_name="PublicIp")
output1 = core.CfnOutput(self,
"MonitoredInstance",
value=output1_1.to_string(),
description="Web Server Public IP to attack")
output2 = core.CfnOutput(
self,
"SSHAlarms",
value=
(f"https://console.aws.amazon.com/cloudwatch/home?region="
f"{core.Aws.REGION}"
f"#/configuration/"
f"#alarmsV2:?search=ssh&alarmStateFilter=ALL&alarmTypeFilter=ALL"
),
description="Check out the cloudwatch Alarms")
output3 = core.CfnOutput(
self,
"SubscribeToNotificationTopic",
value=(f"https://console.aws.amazon.com/sns/v3/home?"
f"{core.Aws.REGION}"
f"#/topic/"
f"{info_sec_ops_topic.topic_arn}"),
description=
"Add your email to subscription and confirm subscription")
output_test_1 = core.CfnOutput(
self,
"ToGenInvalidKeyErrors",
value=
(f"for i in {{1..30}}; do ssh -i $RANDOM ec2-user@{output1_1.to_string()}; sleep 2; done &"
),
description=
"Generates random key names and connects to server 30 times over 60 seconds"
)
output_test_2 = core.CfnOutput(
self,
"ToGenInvalidUserErrors",
value=
(f"for i in {{1..30}}; do ssh ec2-user$RANDOM@{output1_1.to_string()}; sleep 2; done &"
),
description=
"Generates random user names and connects to server 30 times over 60 seconds"
)
"""
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
###
# DynamoDB Table
###
# We store Flight, Hotel and Rental Car bookings in the same table.
#
# For more help with single table DB structures see - https://www.dynamodbbook.com/
# pk - the trip_id e.g. 1234
# sk - bookingtype#booking_id e.g. HOTEL#345634, FLIGHT#574576, PAYMENT#45245
table = dynamo_db.Table(self, "Bookings",
partition_key=dynamo_db.Attribute(name="pk", type=dynamo_db.AttributeType.STRING),
sort_key=dynamo_db.Attribute(name="sk", type=dynamo_db.AttributeType.STRING)
)
###
# Lambda Functions
###
# We need Booking and Cancellation functions for our 3 services
#
# All functions need access to our DynamoDB table above.
# We also need to take payment for this trip
#
# 1) Flights
# 2) Hotel
# 3) Payment
# 1) Flights
reserve_flight_lambda = self.create_lambda(scope=self, lambda_id="reserveFlightLambdaHandler",
handler='flights/reserveFlight.handler', table=table)
confirm_flight_lambda = self.create_lambda(scope=self, lambda_id="confirmFlightLambdaHandler",
handler='flights/confirmFlight.handler', table=table)
cancel_flight_lambda = self.create_lambda(scope=self, lambda_id="cancelFlightLambdaHandler",
handler='flights/cancelFlight.handler', table=table)
# 2) Hotel
reserve_hotel_lambda = self.create_lambda(scope=self, lambda_id="reserveHotelLambdaHandler",
handler='hotel/reserveHotel.handler', table=table)
confirm_hotel_lambda = self.create_lambda(scope=self, lambda_id="confirmHotelLambdaHandler",
handler='hotel/confirmHotel.handler', table=table)
cancel_hotel_lambda = self.create_lambda(scope=self, lambda_id="cancelHotelLambdaHandler",
handler='hotel/cancelHotel.handler', table=table)
# 3) Payment For Holiday
take_payment_lambda = self.create_lambda(scope=self, lambda_id="takePaymentLambdaHandler",
handler='payment/takePayment.handler', table=table)
refund_payment_lambda = self.create_lambda(scope=self, lambda_id="refundPaymentLambdaHandler",
handler='payment/refundPayment.handler', table=table)
###
# Saga Pattern Step Function
###
# Follows a strict order:
# 1) Reserve Flights and Hotel
# 2) Take Payment
# 3) Confirm Flight and Hotel booking
# Our two end states
booking_succeeded = step_fn.Succeed(self, 'We have made your booking!')
booking_failed = step_fn.Fail(self, "Sorry, We Couldn't make the booking")
# 1) Reserve Flights and Hotel
cancel_hotel_reservation = step_fn.Task(self, 'CancelHotelReservation',
task=step_fn_tasks.InvokeFunction(cancel_hotel_lambda),
result_path='$.CancelHotelReservationResult'
).add_retry(max_attempts=3).next(booking_failed)
reserve_hotel = step_fn.Task(self, 'ReserveHotel',
task=step_fn_tasks.InvokeFunction(reserve_hotel_lambda),
result_path='$.ReserveHotelResult'
).add_catch(cancel_hotel_reservation, result_path="$.ReserveHotelError")
cancel_flight_reservation = step_fn.Task(self, 'CancelFlightReservation',
task=step_fn_tasks.InvokeFunction(cancel_flight_lambda),
result_path='$.CancelFlightReservationResult'
).add_retry(max_attempts=3).next(cancel_hotel_reservation)
reserve_flight = step_fn.Task(self, 'ReserveFlight',
task=step_fn_tasks.InvokeFunction(reserve_flight_lambda),
result_path='$.ReserveFlightResult'
).add_catch(cancel_flight_reservation, result_path="$.ReserveFlightError")
# 2) Take Payment
refund_payment = step_fn.Task(self, 'RefundPayment',
task=step_fn_tasks.InvokeFunction(refund_payment_lambda),
result_path='$.RefundPaymentResult'
).add_retry(max_attempts=3).next(cancel_flight_reservation)
take_payment = step_fn.Task(self, 'TakePayment',
task=step_fn_tasks.InvokeFunction(take_payment_lambda),
result_path='$.TakePaymentResult'
).add_catch(refund_payment, result_path="$.TakePaymentError")
# 3) Confirm Flight and Hotel Booking
confirm_hotel = step_fn.Task(self, 'ConfirmHotelBooking',
task=step_fn_tasks.InvokeFunction(confirm_hotel_lambda),
result_path='$.ConfirmHotelBookingResult'
).add_catch(refund_payment, result_path="$.ConfirmHotelBookingError")
confirm_flight = step_fn.Task(self, 'ConfirmFlight',
task=step_fn_tasks.InvokeFunction(confirm_flight_lambda),
result_path='$.ConfirmFlightResult'
).add_catch(refund_payment, result_path="$.ConfirmFlightError")
definition = step_fn.Chain \
.start(reserve_hotel) \
.next(reserve_flight) \
.next(take_payment) \
.next(confirm_hotel) \
.next(confirm_flight) \
.next(booking_succeeded)
saga = step_fn.StateMachine(self, 'BookingSaga', definition=definition, timeout=core.Duration.minutes(5))
# defines an AWS Lambda resource to connect to our API Gateway and kick
# off our step function
saga_lambda = _lambda.Function(self, "sagaLambdaHandler",
runtime=_lambda.Runtime.NODEJS_12_X,
handler="sagaLambda.handler",
code=_lambda.Code.from_asset("lambdas"),
environment={
'statemachine_arn': saga.state_machine_arn
}
)
saga.grant_start_execution(saga_lambda)
# defines an API Gateway REST API resource backed by our "stateMachineLambda" function.
api_gw.LambdaRestApi(self, 'SagaPatternSingleTable',
handler=saga_lambda
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The code that defines your stack goes here
pvt_bkt = _s3.Bucket(self, "s3bucket")
core.Tag.add(pvt_bkt, key="isMonitoredBucket", value="True")
# Lets create a cloudtrail to track s3 data events
s3_data_event_trail = _cloudtrail.Trail(
self,
"s3DataEventTrailId",
is_multi_region_trail=False,
include_global_service_events=False,
enable_file_validation=True)
# Lets capture S3 Data Events only for our bucket- TO REDUCE COST
s3_data_event_trail.add_s3_event_selector(
prefixes=[f"{pvt_bkt.bucket_arn}/"],
include_management_events=True,
read_write_type=_cloudtrail.ReadWriteType.ALL)
# Defines an AWS Lambda resource
"""
with open("lambda_src/make_object_private.py", encoding="utf8") as fp:
make_object_private_fn_handler_code = fp.read()
remediate_object_acl_fn = _lambda.Function(
self,
id='remediateObjAclFn',
function_name="remediate_object_acl_fn",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(make_object_private_fn_handler_code),
handler='index.lambda_handler',
timeout=core.Duration.seconds(10)
)
# Lets add the necessary permission for the lambda function
remediate_object_acl_fn_perms=_iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
"arn:aws:s3:::*",
],
actions=[
"s3:GetObjectAcl",
"s3:PutObjectAcl"
]
)
remediate_object_acl_fn_perms.sid="PutBucketPolicy"
remediate_object_acl_fn.add_to_role_policy( remediate_object_acl_fn_perms )
"""
with open("lambda_src/is_object_private.py", encoding="utf8") as fp:
is_object_private_fn_handler_code = fp.read()
is_object_private_fn = _lambda.Function(
self,
id='isObjPrivateFn',
function_name="is_object_private_fn",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(is_object_private_fn_handler_code),
handler='index.lambda_handler',
timeout=core.Duration.seconds(3))
# Lets add the necessary permission for the lambda function
is_object_private_fn_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
"arn:aws:s3:::*",
],
actions=["s3:GetObjectAcl"])
is_object_private_fn.sid = "CheckObjectAcl"
is_object_private_fn.add_to_role_policy(is_object_private_fn_perms)
with open("lambda_src/make_object_private.py", encoding="utf8") as fp:
make_object_private_fn_handler_code = fp.read()
remediate_object_acl_fn = _lambda.Function(
self,
id='remediateObjAclFn',
function_name="remediate_object_acl_fn",
runtime=_lambda.Runtime.PYTHON_3_7,
code=_lambda.InlineCode(make_object_private_fn_handler_code),
handler='index.lambda_handler',
timeout=core.Duration.seconds(10))
# Lets add the necessary permission for the lambda function
remediate_object_acl_fn_perms = _iam.PolicyStatement(
effect=_iam.Effect.ALLOW,
resources=[
"arn:aws:s3:::*",
],
actions=["s3:PutObjectAcl"])
remediate_object_acl_fn_perms.sid = "PutObjectAcl"
remediate_object_acl_fn.add_to_role_policy(
remediate_object_acl_fn_perms)
info_sec_ops_topic = _sns.Topic(self,
"infoSecOpsTopicId",
display_name="InfoSecTopic",
topic_name="InfoSecOpsTopic")
# Subscribe InfoSecOps Email to topic
info_sec_ops_topic.add_subscription(
_subs.EmailSubscription(global_args.INFO_SEC_OPS_EMAIL))
# Grant Lambda permission to publish to topic
# info_sec_ops_topic.grant_publish(lambda_notifier)
# State Machine for notifying failed ACLs
# Ref: https://docs.aws.amazon.com/cdk/api/latest/docs/aws-stepfunctions-readme.html
###############################################################################
################# STEP FUNCTIONS EXPERIMENTAL CODE - UNSTABLE #################
###############################################################################
is_object_private_task = _sfn.Task(
self,
"isObjectPrivate?",
task=_tasks.InvokeFunction(is_object_private_fn),
result_path="$",
output_path="$")
remediate_object_acl_task = _sfn.Task(
self,
"RemediateObjectAcl",
task=_tasks.InvokeFunction(remediate_object_acl_fn),
result_path="$",
output_path="$")
notify_secops_task = _sfn.Task(
self,
"Notify InfoSecOps",
task=_tasks.PublishToTopic(
info_sec_ops_topic,
integration_pattern=_sfn.ServiceIntegrationPattern.
FIRE_AND_FORGET,
message=_sfn.TaskInput.from_data_at("$.sns_message"),
subject="Object Acl Remediation"))
acl_remediation_failed_task = _sfn.Fail(self,
"Acl Remediation Failed",
cause="Acl Remediation Failed",
error="Check Logs")
acl_compliant_task = _sfn.Succeed(self,
"Object Acl Compliant",
comment="Object Acl is Compliant")
remediate_object_acl_sfn_definition = is_object_private_task\
.next(_sfn.Choice(self, "Is Object Private?")\
.when(_sfn.Condition.boolean_equals("$.is_private", True), acl_compliant_task)\
.when(_sfn.Condition.boolean_equals("$.is_private", False), remediate_object_acl_task\
.next(_sfn.Choice(self, "Object Remediation Complete?")\
.when(_sfn.Condition.boolean_equals("$.status", True),acl_compliant_task)\
.when(_sfn.Condition.boolean_equals("$.status", False), notify_secops_task.next(acl_remediation_failed_task))\
.otherwise(acl_remediation_failed_task)\
)
)
.otherwise(acl_remediation_failed_task)
)
remediate_object_acl_statemachine = _sfn.StateMachine(
self,
"stateMachineId",
definition=remediate_object_acl_sfn_definition,
timeout=core.Duration.minutes(3))
# Cloudwatch Event Triggers
put_object_acl_event_targets = []
"""
put_object_acl_event_targets.append(
_targets.LambdaFunction(
handler=remediate_object_acl_fn
)
)
"""
put_object_acl_event_targets.append(
_targets.SfnStateMachine(
machine=remediate_object_acl_statemachine))
put_object_acl_event_pattern = _events.EventPattern(
source=["aws.s3"],
detail_type=["AWS API Call via CloudTrail"],
detail={
"eventSource": ["s3.amazonaws.com"],
"eventName": ["PutObjectAcl", "PutObject"],
"requestParameters": {
"bucketName": [f"{pvt_bkt.bucket_name}"]
}
})
put_object_acl_event_pattern_rule = _events.Rule(
self,
"putObjectAclEventId",
event_pattern=put_object_acl_event_pattern,
rule_name=f"put_s3_policy_event_{global_args.OWNER}",
enabled=True,
description="Trigger an event for S3 PutObjectAcl or PutObject",
targets=put_object_acl_event_targets)
###########################################
################# OUTPUTS #################
###########################################
output0 = core.CfnOutput(
self,
"SecuirtyAutomationFrom",
value=f"{global_args.SOURCE_INFO}",
description=
"To know more about this automation stack, check out our github page."
)
output1 = core.CfnOutput(
self,
"MonitoredS3Bucket",
value=(f"https://console.aws.amazon.com/s3/buckets/"
f"{pvt_bkt.bucket_name}"),
description=f"S3 Bucket for testing purposes")
output2 = core.CfnOutput(
self,
"Helpercommands",
value=
(f"aws s3api get-object-acl --bucket ${pvt_bkt.bucket_name} --key OBJECT-KEY-NAME"
),
description=
f"Commands to set object to public, Update OBJECT-KEY-NAME to your needs"
)
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# A cache to temporarily hold session info
session_cache_table = aws_dynamodb.Table(
self,
'session_cache_table',
partition_key={
'name': 'code',
'type': aws_dynamodb.AttributeType.STRING
},
billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST,
time_to_live_attribute='expires')
#--
# Secrets
#--------------------#
# Twitter secrets are stored external to this stack
twitter_secret = aws_secretsmanager.Secret.from_secret_attributes(
self,
'twitter_secret',
secret_arn=os.environ['TWITTER_SECRET_ARN'])
#--
# Layers
#--------------------#
# Each of these dependencies is used in 2 or more functions, extracted to layer for ease of use
twitter_layer = aws_lambda.LayerVersion(
self,
'twitter_layer',
code=aws_lambda.AssetCode('layers/twitter_layer'),
compatible_runtimes=[
aws_lambda.Runtime.PYTHON_2_7, aws_lambda.Runtime.PYTHON_3_6
])
boto_layer = aws_lambda.LayerVersion(
self,
'boto_layer',
code=aws_lambda.AssetCode('layers/boto_layer'),
compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_6])
#--
# Functions
#--------------------#
# Handles CRC validation requests from Twitter
twitter_crc_func = aws_lambda.Function(
self,
"twitter_crc_func",
code=aws_lambda.AssetCode('functions/twitter_crc_func'),
handler="lambda.handler",
layers=[twitter_layer],
runtime=aws_lambda.Runtime.PYTHON_2_7,
environment={'twitter_secret': twitter_secret.secret_arn})
# Grant this function the ability to read Twitter credentials
twitter_secret.grant_read(twitter_crc_func.role)
# Handle schedule requests from Twitter
twitter_webhook_func = aws_lambda.Function(
self,
"twitter_webhook_func",
code=aws_lambda.AssetCode('functions/twitter_webhook_func'),
handler="lambda.handler",
layers=[boto_layer, twitter_layer],
runtime=aws_lambda.Runtime.PYTHON_3_6,
environment={'twitter_secret': twitter_secret.secret_arn})
# Grant this function permission to read Twitter credentials
twitter_secret.grant_read(twitter_webhook_func.role)
# Grant this function permission to publish tweets to EventBridge
twitter_webhook_func.add_to_role_policy(
aws_iam.PolicyStatement(actions=["events:PutEvents"],
resources=["*"]))
# Use API Gateway as the webhook endpoint
twitter_api = aws_apigateway.LambdaRestApi(
self, 'twitter_api', handler=twitter_webhook_func, proxy=False)
# Tweets are POSTed to the endpoint
twitter_api.root.add_method('POST')
# Handles twitter CRC validation requests via GET to the webhook
twitter_api.root.add_method(
'GET', aws_apigateway.LambdaIntegration(twitter_crc_func))
# Extract relevant info from the tweet, including session codes
parse_tweet_func = aws_lambda.Function(
self,
"parse_tweet_func",
code=aws_lambda.AssetCode('functions/parse_tweet_func'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6)
# Get session information for requested codes
get_sessions_func = aws_lambda.Function(
self,
"get_sessions_func",
code=aws_lambda.AssetCode('functions/get_sessions_func'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
timeout=core.Duration.seconds(60),
layers=[boto_layer],
environment={
'CACHE_TABLE': session_cache_table.table_name,
'LOCAL_CACHE_TTL':
str(1 * 60 * 60), # Cache sessions locally for 1 hour
'REMOTE_CACHE_TTL': str(12 * 60 * 60)
}) # Cache sessions removely for 12 hours
# This functions needs permissions to read and write to the table
session_cache_table.grant_write_data(get_sessions_func)
session_cache_table.grant_read_data(get_sessions_func)
# Create a schedule without conflicts
create_schedule_func = aws_lambda.Function(
self,
"create_schedule_func",
code=aws_lambda.AssetCode('functions/create_schedule_func'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
timeout=core.Duration.seconds(60))
# Tweet the response to the user
tweet_schedule_func = aws_lambda.Function(
self,
"tweet_schedule_func",
code=aws_lambda.AssetCode('functions/tweet_schedule_func'),
handler="lambda.handler",
layers=[boto_layer, twitter_layer],
runtime=aws_lambda.Runtime.PYTHON_3_6,
environment={'twitter_secret': twitter_secret.secret_arn})
twitter_secret.grant_read(tweet_schedule_func.role)
#--
# States
#--------------------#
# Step 4
tweet_schedule_job = aws_stepfunctions.Task(
self,
'tweet_schedule_job',
task=aws_stepfunctions_tasks.InvokeFunction(tweet_schedule_func))
# Step 3
create_schedule_job = aws_stepfunctions.Task(
self,
'create_schedule_job',
task=aws_stepfunctions_tasks.InvokeFunction(create_schedule_func),
input_path="$.sessions",
result_path="$.schedule")
create_schedule_job.next(tweet_schedule_job)
# Step 2 - Get associated sessions (scrape or cache)
get_sessions_job = aws_stepfunctions.Task(
self,
'get_sessions_job',
task=aws_stepfunctions_tasks.InvokeFunction(get_sessions_func))
# Prepare to get session info in parallel using the Map state
get_sessions_map = aws_stepfunctions.Map(self,
'get_sessions_map',
items_path="$.codes",
result_path="$.sessions")
get_sessions_map.iterator(get_sessions_job)
get_sessions_map.next(create_schedule_job)
# Shortcut if no session codes are supplied
check_num_codes = aws_stepfunctions.Choice(self, 'check_num_codes')
check_num_codes.when(
aws_stepfunctions.Condition.number_greater_than('$.num_codes', 0),
get_sessions_map)
check_num_codes.otherwise(aws_stepfunctions.Succeed(self, "no_codes"))
# Step 1 - Parse incoming tweet and prepare for scheduling
parse_tweet_job = aws_stepfunctions.Task(
self,
'parse_tweet_job',
task=aws_stepfunctions_tasks.InvokeFunction(parse_tweet_func))
parse_tweet_job.next(check_num_codes)
#--
# State Machines
#--------------------#
schedule_machine = aws_stepfunctions.StateMachine(
self, "schedule_machine", definition=parse_tweet_job)
# A rule to filter reInventSched tweet events
reinvent_sched_rule = aws_events.Rule(
self,
"reinvent_sched_rule",
event_pattern={"source": ["reInventSched"]})
# Matching events start the image pipline
reinvent_sched_rule.add_target(
aws_events_targets.SfnStateMachine(
schedule_machine,
input=aws_events.RuleTargetInput.from_event_path("$.detail")))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
email_subscription_parameter = core.CfnParameter(
self,
"EmailSubscriptionParameter",
description="Email Address for Notification Subscription")
email_subscription = email_subscription_parameter.value_as_string
ar1 = accessanalyzer.CfnAnalyzer.ArchiveRuleProperty(
rule_name="test",
filter=[
accessanalyzer.CfnAnalyzer.FilterProperty(
property="principal.AWS", eq=["123456789123"])
])
analyzer = accessanalyzer.CfnAnalyzer(
self,
id="accessanalyzer",
type="ACCOUNT",
tags=[core.CfnTag(key="AccessAnalyzerType", value="ACCOUNT")],
archive_rules=[ar1])
runtime = aws_lambda.Runtime.PYTHON_3_8
boto3_lambda_layer = aws_lambda.LayerVersion(
self,
"Boto3LambdaLayer",
code=aws_lambda.AssetCode("./layers/boto3"),
compatible_runtimes=[runtime],
description="Boto3 Lambda Layer")
context_enrichment = aws_lambda.Function(
self,
"context_enrichment",
runtime=runtime,
handler="app.handler",
code=aws_lambda.AssetCode("./functions/context-enrichment"),
layers=[boto3_lambda_layer])
handler_statement = iam.PolicyStatement(actions=[
"iam:ListRoleTags", "s3:GetBucketTagging", "lambda:ListTags",
"sqs:ListQueueTags", "kms:ListAliases", "kms:ListResourceTags"
],
effect=iam.Effect.ALLOW,
resources=["*"])
context_enrichment.add_to_role_policy(handler_statement)
cmk_key = kms.Key(
self,
"SNSEncryptionAtRestKey",
description="SNS Encryption at rest key",
alias="sns-encryption-at-rest",
enable_key_rotation=True,
)
email_topic = sns.Topic(
self,
"AccessAnalyzerNotificationTopic",
display_name="Access Analyzer Finding Notification Topic",
master_key=cmk_key)
email_topic.add_subscription(
subscriptions.EmailSubscription(email_subscription))
notification = aws_lambda.Function(
self,
"notification",
runtime=runtime,
handler="app.handler",
code=aws_lambda.AssetCode("./functions/notification"),
layers=[boto3_lambda_layer],
environment={"SNS_TOPIC_ARN": email_topic.topic_arn})
notification_statement = iam.PolicyStatement(actions=[
"sns:Publish",
],
effect=iam.Effect.ALLOW,
resources=["*"])
notification.add_to_role_policy(notification_statement)
cmk_key.grant_encrypt_decrypt(notification)
archive_access_analyzer_finding = aws_lambda.Function(
self,
"archive-access-analyzer-finding",
runtime=runtime,
handler="app.handler",
code=aws_lambda.AssetCode(
"./functions/archive-access-analyzer-finding"),
layers=[boto3_lambda_layer])
archive_statement = iam.PolicyStatement(actions=[
"access-analyzer:UpdateFindings",
],
effect=iam.Effect.ALLOW,
resources=["*"])
archive_access_analyzer_finding.add_to_role_policy(archive_statement)
evaluate_access_analyzer_finding = aws_lambda.Function(
self,
"evaluate-access-analyzer-finding",
runtime=runtime,
handler="app.handler",
code=aws_lambda.AssetCode(
"./functions/evaluate-access-analyzer-finding"),
layers=[boto3_lambda_layer])
#https://docs.aws.amazon.com/cdk/api/latest/docs/aws-stepfunctions-readme.html
access_analyzer_handler_task = sfn.Task(
self,
"Context Enrichment",
task=sfn_tasks.InvokeFunction(context_enrichment),
result_path="$.guid",
)
notification_task = sfn.Task(
self,
"Send Notification",
task=sfn_tasks.InvokeFunction(notification),
result_path="$.guid",
)
archive_task = sfn.Task(
self,
"Archive Finding",
task=sfn_tasks.InvokeFunction(archive_access_analyzer_finding),
result_path="$.guid",
)
evaluate_task = sfn.Task(
self,
"Evaluate Risk Level",
task=sfn_tasks.InvokeFunction(evaluate_access_analyzer_finding),
result_path="$.guid",
)
definition=access_analyzer_handler_task. \
next(evaluate_task). \
next(sfn.Choice(self, "Archive?"). \
when(sfn.Condition.string_equals("$.guid.status", "ARCHIVE"), archive_task). \
when(sfn.Condition.string_equals("$.guid.status", "NOTIFY"), notification_task) \
)
state_machine = sfn.StateMachine(
self,
"Access-Analyzer-Automatic-Finding-Archive-State-Machine",
definition=definition,
timeout=core.Duration.minutes(5),
)
#https://docs.aws.amazon.com/step-functions/latest/dg/tutorial-cloudwatch-events-s3.html
access_analyzer_finding_rule = aws_events.Rule(
self,
"AccessAnalzyerFindingActiveEventRule",
description="Access Analyzer Finding Event Active",
enabled=True,
event_pattern=aws_events.EventPattern(
source=["aws.access-analyzer"],
detail_type=["Access Analyzer Finding"],
detail={"status": ["ACTIVE"]}),
targets=[
aws_events_targets.SfnStateMachine(state_machine),
aws_events_targets.LambdaFunction(context_enrichment)
])
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
# The start of the image pipeline
imageBucket = aws_s3.Bucket(self, "imageBucket")
# Capture API activity with a trail
imageBucketTrail = aws_cloudtrail.Trail(self,
"imageBucketTrail",
is_multi_region_trail=False)
# Restrict to S3 data-plane events
imageBucketTrail.add_s3_event_selector(
include_management_events=False,
prefixes=[f"{imageBucket.bucket_arn}/"],
read_write_type=aws_cloudtrail.ReadWriteType.WRITE_ONLY)
# Filter to just PutObject and CopyObject events
imageBucketRule = aws_events.Rule(
self,
"imageBucketRule",
event_pattern={
"source": ["aws.s3"],
"detail": {
"eventSource": ["s3.amazonaws.com"],
"eventName": ["PutObject", "CopyObject"],
"requestParameters": {
"bucketName": [imageBucket.bucket_name]
}
}
})
#--
# Lambda Layers
#--------------------#
opencvLayer = aws_lambda.LayerVersion(
self,
'opencvLayer',
code=aws_lambda.AssetCode('layers/opencvLayer'),
compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_6])
boto3Layer = aws_lambda.LayerVersion(
self,
'boto3Layer',
code=aws_lambda.AssetCode('layers/boto3Layer'),
compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_6])
#--
# Lambda Functions
#--------------------#
# Gather info about an image, name, extension, etc
getImageInfoFunc = aws_lambda.Function(
self,
"getImageInfoFunc",
code=aws_lambda.AssetCode('functions/getImageInfoFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6)
# The home for the website
webBucket = aws_s3.Bucket(self,
"webBucket",
website_index_document='index.html')
# Copy the image to the web bucket
copyImageFunc = aws_lambda.Function(
self,
"copyImageFunc",
code=aws_lambda.AssetCode('functions/copyImageFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
layers=[boto3Layer],
environment={
'OUTPUTBUCKET': webBucket.bucket_name,
'OUTPUTPREFIX': 'images/'
})
# Grant permissions to read from the source and write to the desination
imageBucket.grant_read(copyImageFunc)
webBucket.grant_write(copyImageFunc)
# Create a thumbnail of the image and place in the web bucket
createThumbnailFunc = aws_lambda.Function(
self,
"createThumbnailFunc",
code=aws_lambda.AssetCode('functions/createThumbnailFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
layers=[boto3Layer, opencvLayer],
timeout=core.Duration.seconds(10),
memory_size=256,
environment={
'OUTPUTBUCKET': webBucket.bucket_name,
'OUTPUTPREFIX': 'images/'
})
# Grant permissions to read from the source and write to the desination
imageBucket.grant_read(createThumbnailFunc)
webBucket.grant_write(createThumbnailFunc)
# Store page information
pageTable = aws_dynamodb.Table(
self,
'pageTable',
partition_key={
'name': 'pageName',
'type': aws_dynamodb.AttributeType.STRING
},
billing_mode=aws_dynamodb.BillingMode.PAY_PER_REQUEST,
stream=aws_dynamodb.StreamViewType.NEW_IMAGE)
# Save page and image information
updatePageInfoFunc = aws_lambda.Function(
self,
"updatePageInfoFunc",
code=aws_lambda.AssetCode('functions/updatePageInfoFunc'),
handler="lambda.handler",
runtime=aws_lambda.Runtime.PYTHON_3_6,
layers=[boto3Layer],
environment={
'PAGETABLE': pageTable.table_name,
'PAGEPREFIX': 'posts/'
})
# Grant permissions to write to the page table
pageTable.grant_write_data(updatePageInfoFunc)
imagePipelineDone = aws_stepfunctions.Succeed(self,
"Done processing image")
updatePageInfoJob = aws_stepfunctions.Task(
self,
'Update page info',
task=aws_stepfunctions_tasks.InvokeFunction(updatePageInfoFunc))
updatePageInfoJob.next(imagePipelineDone)
copyImageJob = aws_stepfunctions.Task(
self,
'Copy image',
task=aws_stepfunctions_tasks.InvokeFunction(copyImageFunc))
createThumbnailJob = aws_stepfunctions.Task(
self,
'Create thumbnail',
task=aws_stepfunctions_tasks.InvokeFunction(createThumbnailFunc))
# These tasks can be done in parallel
processImage = aws_stepfunctions.Parallel(self,
'Process image',
result_path="$.images")
processImage.branch(copyImageJob)
processImage.branch(createThumbnailJob)
processImage.next(updatePageInfoJob)
# Results of file extension check
notPng = aws_stepfunctions.Succeed(self, "Not a PNG")
# Verify the file extension
checkForPng = aws_stepfunctions.Choice(self, 'Is a PNG?')
checkForPng.when(
aws_stepfunctions.Condition.string_equals('$.extension', 'png'),
processImage)
checkForPng.otherwise(notPng)
# A single image pipeline job for testing
getImageInfoJob = aws_stepfunctions.Task(
self,
'Get image info',
task=aws_stepfunctions_tasks.InvokeFunction(getImageInfoFunc))
getImageInfoJob.next(checkForPng)
# Configure the image pipeline and starting state
imagePipeline = aws_stepfunctions.StateMachine(
self, "imagePipeline", definition=getImageInfoJob)
# Matching events start the image pipline
imageBucketRule.add_target(
aws_events_targets.SfnStateMachine(
imagePipeline,
input=aws_events.RuleTargetInput.from_event_path(
"$.detail.requestParameters")))
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
email_subscription_parameter = core.CfnParameter(
self,
"EmailSubscriptionParameter",
description="Email Address for Notification Subscription",
allowed_pattern=
'^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$',
min_length=1,
constraint_description="Must be a valid email.")
email_subscription = email_subscription_parameter.value_as_string
#runtime=aws_lambda.Runtime.PYTHON_3_8
boto3_lambda_layer = None
boto3_lambda_layer = self.create_dependencies_layer(
id="boto3layer",
requirements_path="./layers/boto3/requirements.txt",
output_dir="./layers/boto3")
is_inline = False
context_enrichment = self.create_lambda_function(
boto3_lambda_layer, "./functions/context-enrichment",
"context_enrichment", is_inline)
"""
context_enrichment=aws_lambda.Function(
self,
"context_enrichment",
runtime=runtime,
handler="app.handler",
code=aws_lambda.AssetCode("./functions/context-enrichment"),
layers=[boto3_lambda_layer]
)
"""
handler_statement = iam.PolicyStatement(actions=[
"iam:ListRoleTags", "s3:GetBucketTagging", "lambda:ListTags",
"sqs:ListQueueTags", "kms:ListAliases", "kms:ListResourceTags"
],
effect=iam.Effect.ALLOW,
resources=["*"])
context_enrichment.add_to_role_policy(handler_statement)
cmk_key = kms.Key(
self,
"SNSEncryptionAtRestKey",
description="SNS Encryption at rest key",
alias="sns-encryption-at-rest",
enable_key_rotation=True,
)
email_topic = sns.Topic(
self,
"AccessAnalyzerNotificationTopic",
display_name="Access Analyzer Finding Notification Topic",
master_key=cmk_key)
email_topic.add_subscription(
subscriptions.EmailSubscription(email_subscription))
notification = self.create_lambda_function(
boto3_lambda_layer, "./functions/notification", "notification",
is_inline, {"SNS_TOPIC_ARN": email_topic.topic_arn})
"""
notification=aws_lambda.Function(
self,
"notification",
runtime=runtime,
handler="app.handler",
code=aws_lambda.AssetCode("./functions/notification"),
layers=[boto3_lambda_layer],
environment={"SNS_TOPIC_ARN":email_topic.topic_arn}
)
"""
notification_statement = iam.PolicyStatement(actions=[
"sns:Publish",
],
effect=iam.Effect.ALLOW,
resources=["*"])
notification.add_to_role_policy(notification_statement)
cmk_key.grant_encrypt_decrypt(notification)
archive_access_analyzer_finding = self.create_lambda_function(
boto3_lambda_layer, "./functions/archive-access-analyzer-finding",
"archive-access-analyzer-finding", is_inline)
"""
archive_access_analyzer_finding=aws_lambda.Function(
self,
"archive-access-analyzer-finding",
runtime=runtime,
handler="app.handler",
code=aws_lambda.AssetCode("./functions/archive-access-analyzer-finding"),
layers=[boto3_lambda_layer]
)
"""
archive_statement = iam.PolicyStatement(actions=[
"access-analyzer:UpdateFindings",
],
effect=iam.Effect.ALLOW,
resources=["*"])
archive_access_analyzer_finding.add_to_role_policy(archive_statement)
evaluate_access_analyzer_finding = self.create_lambda_function(
boto3_lambda_layer, "./functions/evaluate-access-analyzer-finding",
"evaluate-access-analyzer-finding", is_inline)
"""
evaluate_access_analyzer_finding=aws_lambda.Function(
self,
"evaluate-access-analyzer-finding",
runtime=runtime,
handler="app.handler",
code=aws_lambda.AssetCode("./functions/evaluate-access-analyzer-finding"),
layers=[boto3_lambda_layer]
)
"""
#https://docs.aws.amazon.com/cdk/api/latest/docs/aws-stepfunctions-readme.html
access_analyzer_handler_task = sfn.Task(
self,
"Context Enrichment",
task=sfn_tasks.InvokeFunction(context_enrichment),
result_path="$.guid",
)
notification_task = sfn.Task(
self,
"Send Notification",
task=sfn_tasks.InvokeFunction(notification),
result_path="$.guid",
)
archive_task = sfn.Task(
self,
"Archive Finding",
task=sfn_tasks.InvokeFunction(archive_access_analyzer_finding),
result_path="$.guid",
)
evaluate_task = sfn.Task(
self,
"Evaluate Risk Level",
task=sfn_tasks.InvokeFunction(evaluate_access_analyzer_finding),
result_path="$.guid",
)
definition=access_analyzer_handler_task. \
next(evaluate_task). \
next(sfn.Choice(self, "Archive?"). \
when(sfn.Condition.string_equals("$.guid.status", "ARCHIVE"), archive_task). \
when(sfn.Condition.string_equals("$.guid.status", "NOTIFY"), notification_task) \
)
state_machine = sfn.StateMachine(
self,
"Access-Analyzer-Automatic-Finding-Archive-State-Machine",
definition=definition,
timeout=core.Duration.minutes(5),
)
#https://docs.aws.amazon.com/step-functions/latest/dg/tutorial-cloudwatch-events-s3.html
access_analyzer_finding_rule = aws_events.Rule(
self,
"AccessAnalzyerFindingActiveEventRule",
description="Access Analyzer Finding Event Active",
enabled=True,
event_pattern=aws_events.EventPattern(
source=["aws.access-analyzer"],
detail_type=["Access Analyzer Finding"],
detail={"status": ["ACTIVE"]}),
targets=[
aws_events_targets.SfnStateMachine(state_machine),
aws_events_targets.LambdaFunction(context_enrichment)
])
