def get_or_build(scope: core.Construct) -> aws_lambda.Function: code = aws_lambda.Code.from_asset( _lambda_path('emr_utilities/fail_if_cluster_running')) stack = core.Stack.of(scope) layer = EMRConfigUtilsLayerBuilder.get_or_build(scope) lambda_function = stack.node.try_find_child('FailIfClusterRunning') if lambda_function is None: lambda_function = aws_lambda.Function( stack, 'FailIfClusterRunning', code=code, handler='lambda_source.handler', runtime=aws_lambda.Runtime.PYTHON_3_7, timeout=core.Duration.minutes(1), layers=[layer], initial_policy=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['elasticmapreduce:ListClusters'], resources=['*']) ]) BaseBuilder.tag_construct(lambda_function) return lambda_function
def get_or_build(scope: core.Construct) -> aws_lambda.LayerVersion: code = aws_lambda.Code.from_asset( _lambda_path('layers/emr_config_utils')) stack = core.Stack.of(scope) layer = stack.node.try_find_child('EMRConfigUtilsLayer') if layer is None: layer = aws_lambda.LayerVersion( stack, 'EMRConfigUtilsLayer', layer_version_name='EMRLaunch_EMRUtilities_EMRConfigUtilsLayer', code=code, compatible_runtimes=[aws_lambda.Runtime.PYTHON_3_7], description='EMR configuration utility functions') BaseBuilder.tag_construct(layer) return layer
def get_or_build(scope: core.Construct, event_rule: events.Rule) -> aws_lambda.Function: code = aws_lambda.Code.from_asset( _lambda_path('emr_utilities/check_cluster_status')) stack = core.Stack.of(scope) layer = EMRConfigUtilsLayerBuilder.get_or_build(scope) lambda_function = stack.node.try_find_child('CheckClusterStatus') if lambda_function is None: lambda_function = aws_lambda.Function( stack, 'CheckClusterStatus', code=code, handler='lambda_source.handler', runtime=aws_lambda.Runtime.PYTHON_3_7, timeout=core.Duration.minutes(1), layers=[layer], initial_policy=[ iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=[ 'states:SendTaskSuccess', 'states:SendTaskHeartbeat', 'states:SendTaskFailure' ], resources=['*']), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['elasticmapreduce:DescribeCluster'], resources=['*']), iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=[ 'events:ListTargetsByRule', 'events:DisableRule', 'events:RemoveTargets' ], resources=[event_rule.rule_arn]) ]) BaseBuilder.tag_construct(lambda_function) lambda_function.add_permission( 'EventRulePermission', principal=iam.ServicePrincipal('events.amazonaws.com'), action='lambda:InvokeFunction', source_arn=event_rule.rule_arn) return lambda_function
def get_or_build(scope: core.Construct) -> aws_lambda.Function: code = aws_lambda.Code.from_asset( _lambda_path('emr_utilities/override_cluster_configs')) stack = core.Stack.of(scope) layer = EMRConfigUtilsLayerBuilder.get_or_build(scope) lambda_function = stack.node.try_find_child('OverrideClusterConfigs') if lambda_function is None: lambda_function = aws_lambda.Function( stack, 'OverrideClusterConfigs', code=code, handler='lambda_source.handler', runtime=aws_lambda.Runtime.PYTHON_3_7, timeout=core.Duration.minutes(1), layers=[layer]) BaseBuilder.tag_construct(lambda_function) return lambda_function
def get_or_build(scope: core.Construct, roles: emr_roles.EMRRoles, event_rule: events.Rule) -> aws_lambda.Function: code = aws_lambda.Code.from_asset( _lambda_path('emr_utilities/run_job_flow')) stack = core.Stack.of(scope) layer = EMRConfigUtilsLayerBuilder.get_or_build(scope) lambda_function = stack.node.try_find_child('RunJobFlow') if lambda_function is None: lambda_function = aws_lambda.Function( stack, 'RunJobFlow', code=code, handler='lambda_source.handler', runtime=aws_lambda.Runtime.PYTHON_3_7, timeout=core.Duration.minutes(1), layers=[layer], initial_policy=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['elasticmapreduce:RunJobFlow'], resources=['*']), iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=['iam:PassRole'], resources=[ roles.service_role.role_arn, roles.instance_role.role_arn, roles.autoscaling_role.role_arn ]), iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=['states:SendTaskSuccess'], resources=['*']), iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['events:EnableRule', 'events:PutTargets'], resources=[event_rule.rule_arn]) ]) BaseBuilder.tag_construct(lambda_function) return lambda_function
def build(scope: core.Construct, profile_namespace: str, profile_name: str, configuration_namespace: str, configuration_name: str) -> aws_lambda.Function: code = aws_lambda.Code.from_asset( _lambda_path('emr_utilities/load_cluster_configuration')) stack = core.Stack.of(scope) layer = EMRConfigUtilsLayerBuilder.get_or_build(scope) lambda_function = aws_lambda.Function( scope, 'LoadClusterConfiguration', code=code, handler='lambda_source.handler', runtime=aws_lambda.Runtime.PYTHON_3_7, timeout=core.Duration.minutes(1), layers=[layer], initial_policy=[ iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['ssm:GetParameter'], resources=[ stack.format_arn( partition=stack.partition, service='ssm', resource= 'parameter/emr_launch/cluster_configurations/' f'{configuration_namespace}/{configuration_name}'), stack.format_arn( partition=stack.partition, service='ssm', resource='parameter/emr_launch/emr_profiles/' f'{profile_namespace}/{profile_name}') ]) ]) BaseBuilder.tag_construct(lambda_function) return lambda_function
def build(scope: core.Construct, id: str, *, roles: emr_roles.EMRRoles, kerberos_attributes_secret: Optional[ secretsmanager.Secret] = None, secret_configurations: Optional[Dict[ str, secretsmanager.Secret]] = None, cluster_configuration_path: str = '$.ClusterConfiguration', result_path: Optional[str] = None, output_path: Optional[str] = None, wait_for_cluster_start: bool = True) -> sfn.Task: # We use a nested Construct to avoid collisions with Lambda and Task ids construct = core.Construct(scope, id) event_rule = core.Stack.of(scope).node.try_find_child('EventRule') if event_rule is None: event_rule = events.Rule(construct, 'EventRule', enabled=False, schedule=events.Schedule.rate( core.Duration.minutes(1))) BaseBuilder.tag_construct(event_rule) run_job_flow_lambda = emr_lambdas.RunJobFlowBuilder.get_or_build( construct, roles, event_rule) check_cluster_status_lambda = emr_lambdas.CheckClusterStatusBuilder.get_or_build( construct, event_rule) if kerberos_attributes_secret: run_job_flow_lambda.add_to_role_policy( iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['secretsmanager:GetSecretValue'], resources=[f'{kerberos_attributes_secret.secret_arn}*'])) if secret_configurations is not None: for secret in secret_configurations.values(): run_job_flow_lambda.add_to_role_policy( iam.PolicyStatement( effect=iam.Effect.ALLOW, actions=['secretsmanager:GetSecretValue'], resources=[f'{secret.secret_arn}*'])) return sfn.Task( construct, 'Start EMR Cluster (with Secrets)', output_path=output_path, result_path=result_path, task=sfn_tasks.RunLambdaTask( run_job_flow_lambda, integration_pattern=sfn.ServiceIntegrationPattern. WAIT_FOR_TASK_TOKEN, payload=sfn.TaskInput.from_object({ 'ExecutionInput': sfn.TaskInput.from_context_at('$$.Execution.Input').value, 'ClusterConfiguration': sfn.TaskInput.from_data_at( cluster_configuration_path).value, 'TaskToken': sfn.Context.task_token, 'CheckStatusLambda': check_cluster_status_lambda.function_arn, 'RuleName': event_rule.rule_name, 'FireAndForget': not wait_for_cluster_start })))