def create_monitoring_schedule( scope, blueprint_bucket, assets_bucket, baseline_job_output_location, baseline_job_name, monitoring_schedual_name, monitoring_output_location, schedule_expression, endpoint_name, instance_type, instance_volume_size, max_runtime_seconds, monitoring_type, stack_name, ): """ create_monitoring_schedule creates a model monitoring job in a lambda invoked codepipeline action :scope: CDK Construct scope that's needed to create CDK resources :blueprint_bucket: CDK object of the blueprint bucket that contains resources for BYOM pipeline :assets_bucket: the bucket cdk object where pipeline assets are stored :baseline_job_output_location: S3 prefix in the S3 assets bucket to store the output of the job :baseline_job_name: name of the baseline job :monitoring_schedual_name: name of the monitoring job to be created :schedule_expression cron job expression :endpoint_name: name of the deployed SageMaker endpoint to be monitored :instance_type: compute instance type for the baseline job, in the form of a CDK CfnParameter object :instance_volume_size: volume size of the EC2 instance :monitoring_type: type of monitoring to be created :max_runtime_seconds: max time the job is allowd to run :stack_name: name of the model monitoring satck :return: codepipeline action in a form of a CDK object that can be attached to a codepipeline stage """ create_monitoring_schedule_policy = iam.PolicyStatement( actions=[ "sagemaker:DescribeEndpointConfig", "sagemaker:DescribeEndpoint", "sagemaker:CreateMonitoringSchedule", "sagemaker:DescribeMonitoringSchedule", "sagemaker:StopMonitoringSchedule", "sagemaker:DeleteMonitoringSchedule", "sagemaker:DescribeProcessingJob", ], resources=[ (f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:" f"endpoint/{endpoint_name.value_as_string}*"), (f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:" f"endpoint-config/{endpoint_name.value_as_string}*"), (f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:" f"monitoring-schedule/{monitoring_schedual_name.value_as_string}" ), (f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:" f"processing-job/{baseline_job_name.value_as_string}"), ], ) s3_policy = iam.PolicyStatement( actions=[ "s3:ListBucket", "s3:GetObject", "s3:PutObject", ], resources=[ assets_bucket.bucket_arn, assets_bucket.arn_for_objects("*"), ], ) sagemaker_logs_policy = iam.PolicyStatement( actions=[ "cloudwatch:PutMetricData", "logs:CreateLogStream", "logs:PutLogEvents", "logs:CreateLogGroup", "logs:DescribeLogStreams", ], resources=["*"], ) # create sagemaker role sagemaker_role = iam.Role( scope, "create_monitoring_scheduale_sagemaker_role", assumed_by=iam.ServicePrincipal("sagemaker.amazonaws.com"), description= "Role that is create sagemaker model Lambda function assumes to create a model in the pipeline.", ) # create a trust relation to assume the Role sagemaker_role.add_to_policy( iam.PolicyStatement(actions=["sts:AssumeRole"], resources=[sagemaker_role.role_arn])) # creating a role so that this lambda can create a baseline job lambda_role = iam.Role( scope, "create_monitoring_scheduale_role", assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), description= "Role that is create_data_baseline_job Lambda function assumes to create a baseline job in the pipeline.", ) sagemaker_role.add_to_policy(create_monitoring_schedule_policy) sagemaker_role.add_to_policy(sagemaker_logs_policy) sagemaker_role.add_to_policy(s3_policy) sagemaker_role_nodes = sagemaker_role.node.find_all() sagemaker_role_nodes[ 2].node.default_child.cfn_options.metadata = suppress_pipeline_policy( ) lambda_role.add_to_policy( iam.PolicyStatement(actions=["iam:PassRole"], resources=[sagemaker_role.role_arn])) lambda_role.add_to_policy(create_monitoring_schedule_policy) lambda_role.add_to_policy(s3_policy) add_logs_policy(lambda_role) # defining the lambda function that gets invoked in this stage create_moniroring_schedule_lambda = lambda_.Function( scope, "create_moniroring_schedule", runtime=lambda_.Runtime.PYTHON_3_8, handler="main.handler", role=lambda_role, code=lambda_.Code.from_bucket( blueprint_bucket, "blueprints/byom/lambdas/create_model_monitoring_schedule.zip"), environment={ "BASELINE_JOB_NAME": baseline_job_name.value_as_string, "BASELINE_JOB_OUTPUT_LOCATION": baseline_job_output_location.value_as_string, "ASSETS_BUCKET": assets_bucket.bucket_name, "SAGEMAKER_ENDPOINT_NAME": f"{endpoint_name.value_as_string}", "MONITORING_SCHEDULE_NAME": monitoring_schedual_name.value_as_string, "MONITORING_OUTPUT_LOCATION": monitoring_output_location.value_as_string, "SCHEDULE_EXPRESSION": schedule_expression.value_as_string, "INSTANCE_TYPE": instance_type.value_as_string, "INSTANCE_VOLUME_SIZE": instance_volume_size.value_as_string, "MAX_RUNTIME_SECONDS": max_runtime_seconds.value_as_string, "ROLE_ARN": sagemaker_role.role_arn, "MONITORING_TYPE": monitoring_type.value_as_string, "STACK_NAME": stack_name, "LOG_LEVEL": "INFO", }, timeout=core.Duration.minutes(10), ) create_moniroring_schedule_lambda.node.default_child.cfn_options.metadata = suppress_cloudwatch_policy( ) role_child_nodes = create_moniroring_schedule_lambda.role.node.find_all() role_child_nodes[ 2].node.default_child.cfn_options.metadata = suppress_pipeline_policy( ) # Create codepipeline action create_moniroring_schedule_action = codepipeline_actions.LambdaInvokeAction( action_name="create_monitoring_schedule", inputs=[], outputs=[], variables_namespace="monitoring_schedule", lambda_=create_moniroring_schedule_lambda, run_order=2, # this runs second in the deploy stage ) return (create_moniroring_schedule_lambda.function_arn, create_moniroring_schedule_action)
def batch_transform( scope, blueprint_bucket, assets_bucket, model_name, inference_instance, batch_inference_data, sm_layer, ): """ batch_transform creates a sagemaker batch transform job in a lambda invoked codepipeline action :scope: CDK Construct scope that's needed to create CDK resources :blueprint_bucket: CDK object of the blueprint bucket that contains resources for BYOM pipeline :assets_bucket: the bucket cdk object where pipeline assets are stored :model_name: name of the sagemaker model to be created, in the form of a CDK CfnParameter object :inference_instance: compute instance type for the sagemaker inference endpoint, in the form of a CDK CfnParameter object :batch_inference_data: location of the batch inference data in assets bucket, in the form of a CDK CfnParameter object :is_batch_transform: a CDK CfnCondition object that says if inference type is batch transform or not :sm_layer: sagemaker lambda layer :return: codepipeline action in a form of a CDK object that can be attached to a codepipeline stage """ batch_transform_policy = iam.PolicyStatement( actions=[ "sagemaker:CreateTransformJob", "s3:ListBucket", "s3:GetObject", "s3:PutObject", ], resources=[ assets_bucket.bucket_arn, assets_bucket.arn_for_objects("*"), (f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:" f"transform-job/{model_name.value_as_string}-*"), ], ) lambda_role = iam.Role( scope, "batch_transform_lambda_role", assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), description= ("Role that creates a lambda function assumes to create a sagemaker batch transform " "job in the aws mlops pipeline."), ) lambda_role.add_to_policy(batch_transform_policy) lambda_role.add_to_policy(codepipeline_policy()) add_logs_policy(lambda_role) # defining batch transform lambda function batch_transform = lambda_.Function( scope, "batch_transform", runtime=lambda_.Runtime.PYTHON_3_8, handler="main.handler", layers=[sm_layer], role=lambda_role, code=lambda_.Code.from_bucket( blueprint_bucket, "blueprints/byom/lambdas/batch_transform.zip"), environment={ "model_name": model_name.value_as_string, "inference_instance": inference_instance.value_as_string, "assets_bucket": assets_bucket.bucket_name, "batch_inference_data": batch_inference_data.value_as_string, "LOG_LEVEL": "INFO", }, ) batch_transform.node.default_child.cfn_options.metadata = suppress_cloudwatch_policy( ) role_child_nodes = batch_transform.role.node.find_all() role_child_nodes[ 2].node.default_child.cfn_options.metadata = suppress_pipeline_policy( ) batch_transform_action = codepipeline_actions.LambdaInvokeAction( action_name="batch_transform", inputs=[], outputs=[], variables_namespace="batch_transform", lambda_=batch_transform, run_order=2, # this runs second in the deploy stage ) return (batch_transform.function_arn, batch_transform_action)
def create_model( scope, blueprint_bucket, assets_bucket, model_name, model_artifact_location, custom_container, model_framework, model_framework_version, container_uri, sm_layer, ): """ create_model creates a sagemaker model in a lambda invoked codepipeline action :scope: CDK Construct scope that's needed to create CDK resources :blueprint_bucket: CDK object of the blueprint bucket that contains resources for BYOM pipeline :assets_bucket: the bucket cdk object where pipeline assets are stored :model_name: name of the sagemaker model to be created, in the form of a CDK CfnParameter object :model_artifact_location: path to the model artifact in the S3 bucket: assets_bucket :custom_container: whether to the model is a custom algorithm or a sagemaker algorithmm, in the form of a CDK CfnParameter object :model_framework: name of the framework if the model is a sagemaker algorithm, in the form of a CDK CfnParameter object :model_framework_version: version of the framework if the model is a sagemaker algorithm, in the form of a CDK CfnParameter object :container_uri: URI for the container registry that stores the model if the model is a custom algorithm :sm_layer: sagemaker lambda layer :return: codepipeline action in a form of a CDK object that can be attached to a codepipeline stage """ create_model_policy = iam.PolicyStatement( actions=[ "sagemaker:CreateModel", "sagemaker:DescribeModel", "sagemaker:DeleteModel", ], resources=[ # Lambda that uses this polict requires access to all objects in the assets bucket f"arn:{core.Aws.PARTITION}:s3:::{assets_bucket.bucket_name}/*", (f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}" f":model/{model_name.value_as_string}"), ], ) s3_policy = iam.PolicyStatement( actions=[ "s3:GetObject", "s3:PutObject", "s3:ListBucket", ], resources=[ assets_bucket.arn_for_objects("*"), assets_bucket.bucket_arn ], ) # creating this policy for sagemaker create endpoint in custom model ecr_policy = iam.PolicyStatement( actions=[ "ecr:BatchGetImage", "ecr:BatchCheckLayerAvailability", "ecr:DescribeImages", "ecr:DescribeRepositories", "ecr:GetDownloadUrlForLayer", ], resources=[ f"arn:{core.Aws.PARTITION}:ecr:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:repository/awsmlopsmodels*" ], ) ecr_token_policy = iam.PolicyStatement( actions=["ecr:GetAuthorizationToken"], resources=[ "*" ], # GetAuthorizationToken can not be bound to resources other than * ) # creating a role for the lambda function so that it can create a model in sagemaker sagemaker_role = iam.Role( scope, "create_model_sagemaker_role", assumed_by=iam.ServicePrincipal("sagemaker.amazonaws.com"), description= "Role that is create sagemaker model Lambda function assumes to create a model in the pipeline.", ) lambda_role = iam.Role( scope, "create_model_lambda_role", assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), description= "Role that is create sagemaker model Lambda function assumes to create a model in the pipeline.", ) sagemaker_role.add_to_policy(create_model_policy) sagemaker_role.add_to_policy(s3_policy) sagemaker_role.add_to_policy(ecr_policy) sagemaker_role.add_to_policy(ecr_token_policy) sagemaker_role_nodes = sagemaker_role.node.find_all() sagemaker_role_nodes[ 2].node.default_child.cfn_options.metadata = suppress_ecr_policy() lambda_role.add_to_policy( iam.PolicyStatement(actions=["iam:PassRole"], resources=[sagemaker_role.role_arn])) lambda_role.add_to_policy(create_model_policy) lambda_role.add_to_policy(s3_policy) add_logs_policy(lambda_role) # defining the lambda function that gets invoked by codepipeline in this step create_sagemaker_model = lambda_.Function( scope, "create_sagemaker_model", runtime=lambda_.Runtime.PYTHON_3_8, handler="main.handler", timeout=core.Duration.seconds(60), code=lambda_.Code.from_bucket( blueprint_bucket, "blueprints/byom/lambdas/create_sagemaker_model.zip"), layers=[sm_layer], role=lambda_role, environment={ "custom_container": custom_container.value_as_string, "model_framework": model_framework.value_as_string, "model_framework_version": model_framework_version.value_as_string, "model_name": model_name.value_as_string, "model_artifact_location": assets_bucket.s3_url_for_object( model_artifact_location.value_as_string), "create_model_role_arn": sagemaker_role.role_arn, "container_uri": container_uri, "LOG_LEVEL": "INFO", }, ) create_sagemaker_model.node.default_child.cfn_options.metadata = suppress_cloudwatch_policy( ) role_child_nodes = create_sagemaker_model.role.node.find_all() role_child_nodes[ 2].node.default_child.cfn_options.metadata = suppress_pipeline_policy( ) # creating the codepipeline action that invokes create model lambda create_sagemaker_model_action = codepipeline_actions.LambdaInvokeAction( action_name="create_sagemaker_model", inputs=[], outputs=[], lambda_=create_sagemaker_model, run_order=1, # runs first in the Deploy stage ) return (create_sagemaker_model.function_arn, create_sagemaker_model_action)
def create_endpoint(scope, blueprint_bucket, assets_bucket, model_name, inference_instance): """ create_endpoint creates a sagemaker inference endpoint in a lambda invoked codepipeline action :scope: CDK Construct scope that's needed to create CDK resources :blueprint_bucket: CDK object of the blueprint bucket that contains resources for BYOM pipeline :assets_bucket: the bucket cdk object where pipeline assets are stored :model_name: name of the sagemaker model to be created, in the form of a CDK CfnParameter object :inference_instance: compute instance type for the sagemaker inference endpoint, in the form of a CDK CfnParameter object :is_realtime_inference: a CDK CfnCondition object that says if inference type is realtime or not :return: codepipeline action in a form of a CDK object that can be attached to a codepipeline stage """ create_endpoint_policy = iam.PolicyStatement( actions=[ "sagemaker:CreateEndpoint", "sagemaker:CreateEndpointConfig", "sagemaker:DeleteEndpointConfig", "sagemaker:DescribeEndpointConfig", "sagemaker:DescribeEndpoint", ], resources=[ (f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:" f"endpoint/{model_name.value_as_string}-endpoint"), (f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:" f"endpoint-config/{model_name.value_as_string}-endpoint-config"), ], ) # creating a role so that this lambda can create a sagemaker endpoint and endpoint config lambda_role = iam.Role( scope, "create_endpoint_lambda_role", assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), description= "Role that is create sagemaker model Lambda function assumes to create a model in the pipeline.", ) lambda_role.add_to_policy(create_endpoint_policy) add_logs_policy(lambda_role) # defining the lambda function that gets invoked in this stage create_sagemaker_endpoint = lambda_.Function( scope, "create_sagemaker_endpoint", runtime=lambda_.Runtime.PYTHON_3_8, handler="main.handler", role=lambda_role, code=lambda_.Code.from_bucket( blueprint_bucket, "blueprints/byom/lambdas/create_sagemaker_endpoint.zip"), environment={ "model_name": model_name.value_as_string, "inference_instance": inference_instance.value_as_string, "assets_bucket": assets_bucket.bucket_name, "LOG_LEVEL": "INFO", }, timeout=core.Duration.minutes(10), ) create_sagemaker_endpoint.node.default_child.cfn_options.metadata = suppress_cloudwatch_policy( ) role_child_nodes = create_sagemaker_endpoint.role.node.find_all() role_child_nodes[ 2].node.default_child.cfn_options.metadata = suppress_pipeline_policy( ) # create_endpoint_action = core.Fn.condition_if("isRealtimeInference", create_endpoint_action = codepipeline_actions.LambdaInvokeAction( action_name="create_sagemaker_endpoint", inputs=[], outputs=[], variables_namespace="sagemaker_endpoint", lambda_=create_sagemaker_endpoint, run_order=2, # this runs second in the deploy stage ) return (create_sagemaker_endpoint.function_arn, create_endpoint_action)
def build_action(scope, source_output): """ build_action configures a codepipeline action that takes Dockerfile and creates a container image :scope: CDK Construct scope that's needed to create CDK resources :source_output: output of the source stage in codepipeline :is_custom_container: a CDK CfnCondition object, if true, it creates resources for the pipeline action :return: codepipeline action in a form of a CDK object that can be attached to a codepipeline stage """ model_containers = ecr.Repository(scope, "awsmlopsmodels") codebuild_role = iam.Role( scope, "codebuildRole", assumed_by=iam.ServicePrincipal("codebuild.amazonaws.com")) codebuild_policy = iam.PolicyStatement(actions=[ "ecr:BatchCheckLayerAvailability", "ecr:CompleteLayerUpload", "ecr:InitiateLayerUpload", "ecr:PutImage", "ecr:UploadLayerPart", ], resources=[ model_containers.repository_arn, ]) codebuild_role.add_to_policy(codebuild_policy) codebuild_role.add_to_policy( iam.PolicyStatement(actions=["ecr:GetAuthorizationToken"], resources=["*"])) codebuild_role_child_nodes = codebuild_role.node.find_all() codebuild_role_child_nodes[ 3].cfn_options.metadata = suppress_pipeline_policy() # codebuild setup for build stage container_factory_project = codebuild.PipelineProject( scope, "Container_Factory", build_spec=codebuild.BuildSpec.from_object({ "version": "0.2", "phases": { "pre_build": { "commands": [ "echo Logging in to Amazon ECR...", "aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com", "find . -iname \"serve\" -exec chmod 777 \"{}\" \\;", "find . -iname \"train\" -exec chmod 777 \"{}\" \\;", ] }, "build": { "commands": [ "echo Build started on `date`", "echo Building the Docker image...", "docker build -t $IMAGE_REPO_NAME:$IMAGE_TAG .", "docker tag $IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG" ] }, "post_build": { "commands": [ "echo Build completed on `date`", "echo Pushing the Docker image...", "docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG" ] } }, }), environment=codebuild.BuildEnvironment( build_image=codebuild.LinuxBuildImage.STANDARD_4_0, compute_type=codebuild.ComputeType.SMALL, environment_variables={ "AWS_DEFAULT_REGION": { "value": core.Aws.REGION }, "AWS_ACCOUNT_ID": { "value": core.Aws.ACCOUNT_ID }, "IMAGE_REPO_NAME": { "value": model_containers.repository_name }, "IMAGE_TAG": { "value": "latest" } }, privileged=True), role=codebuild_role) build_action_definition = codepipeline_actions.CodeBuildAction( action_name="CodeBuild", project=container_factory_project, input=source_output, outputs=[codepipeline.Artifact()]) container_uri = f'{core.Aws.ACCOUNT_ID}.dkr.ecr.{core.Aws.REGION}.amazonaws.com/{model_containers.repository_name}:latest' return build_action_definition, container_uri
def build_action(scope, ecr_repository_name, image_tag, source_output): """ build_action configures a codepipeline action with repository name and tag :scope: CDK Construct scope that's needed to create CDK resources :ecr_repository_name: name of Amazon ECR repository where the image will be stored :image_tag: docker image tag to be assigned. :return: codepipeline action in a form of a CDK object that can be attached to a codepipeline stage """ codebuild_role = iam.Role( scope, "codebuildRole", assumed_by=iam.ServicePrincipal("codebuild.amazonaws.com")) codebuild_policy = iam.PolicyStatement( actions=[ "ecr:BatchCheckLayerAvailability", "ecr:CompleteLayerUpload", "ecr:InitiateLayerUpload", "ecr:PutImage", "ecr:UploadLayerPart", ], resources=[ f"arn:{core.Aws.PARTITION}:ecr:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:repository/{ecr_repository_name}", ], ) codebuild_role.add_to_policy(codebuild_policy) codebuild_role.add_to_policy( iam.PolicyStatement(actions=["ecr:GetAuthorizationToken"], resources=["*"])) codebuild_role_child_nodes = codebuild_role.node.find_all() codebuild_role_child_nodes[ 3].cfn_options.metadata = suppress_pipeline_policy() # codebuild setup for build stage container_factory_project = codebuild.PipelineProject( scope, "Container_Factory", build_spec=codebuild.BuildSpec.from_object({ "version": "0.2", "phases": { "pre_build": { "commands": [ "echo Logging in to Amazon ECR...", ("aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS " "--password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com" ), 'find . -iname "serve" -exec chmod 777 "{}" \\;', 'find . -iname "train" -exec chmod 777 "{}" \\;', ] }, "build": { "commands": [ "echo Build started on `date`", "echo Building the Docker image...", "docker build -t $IMAGE_REPO_NAME:$IMAGE_TAG .", ("docker tag $IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_ID.dkr.ecr." "$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG" ), ] }, "post_build": { "commands": [ "echo Build completed on `date`", "echo Pushing the Docker image...", ("docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/" "$IMAGE_REPO_NAME:$IMAGE_TAG"), ] }, }, }), environment=codebuild.BuildEnvironment( build_image=codebuild.LinuxBuildImage.STANDARD_4_0, compute_type=codebuild.ComputeType.SMALL, environment_variables={ "AWS_DEFAULT_REGION": { "value": core.Aws.REGION }, "AWS_ACCOUNT_ID": { "value": core.Aws.ACCOUNT_ID }, "IMAGE_REPO_NAME": { "value": ecr_repository_name }, "IMAGE_TAG": { "value": image_tag }, }, privileged=True, ), role=codebuild_role, ) build_action_definition = codepipeline_actions.CodeBuildAction( action_name="CodeBuild", project=container_factory_project, input=source_output, outputs=[codepipeline.Artifact()], ) container_uri = f"{core.Aws.ACCOUNT_ID}.dkr.ecr.{core.Aws.REGION}.amazonaws.com/{ecr_repository_name}:{image_tag}" return build_action_definition, container_uri
def create_stackset_action( scope, # NOSONAR:S107 this function is designed to take many arguments action_name, blueprint_bucket, source_output, artifact, template_file, stage_params_file, accound_ids, org_ids, regions, assets_bucket, stack_name, ): """ create_stackset_action an invokeLambda action to be added to AWS Codepipeline stage :scope: CDK Construct scope that's needed to create CDK resources :action_name: name of the StackSet action :blueprint_bucket: CDK object of the blueprint bucket that contains resources for BYOM pipeline :source_output: CDK object of the Source action's output :artifact: name of the input aritifcat to the StackSet action :template_file: name of the Cloudformation template to be deployed :stage_params_file: name of the template parameters for the satge :accound_ids: list of AWS acounts where the stack with be deployed :org_ids: list of AWS orginizational ids where the stack with be deployed :regions: list of regions where the stack with be deployed :assets_bucket: the bucket cdk object where pipeline assets are stored :stack_name: name of the stack to be deployed :return: codepipeline invokeLambda action in a form of a CDK object that can be attached to a codepipeline stage """ # creating a role so that this lambda can create a baseline job lambda_role = create_service_role( scope, f"{action_name}_role", lambda_service, "The role that is assumed by create_update_cf_stackset Lambda function.", ) # make the stackset name unique stack_name = f"{stack_name}-{str(uuid.uuid4())[:8]}" # cloudformation stackset permissions cloudformation_stackset_permissions = cloudformation_stackset_policy( stack_name) cloudformation_stackset_instances_permissions = cloudformation_stackset_instances_policy( stack_name) lambda_role.add_to_policy(cloudformation_stackset_permissions) lambda_role.add_to_policy(cloudformation_stackset_instances_permissions) add_logs_policy(lambda_role) # defining the lambda function that gets invoked in this stage create_update_cf_stackset_lambda = lambda_.Function( scope, f"{action_name}_stackset_lambda", runtime=lambda_.Runtime.PYTHON_3_8, handler="main.lambda_handler", role=lambda_role, code=lambda_.Code.from_bucket( blueprint_bucket, "blueprints/byom/lambdas/create_update_cf_stackset.zip"), timeout=core.Duration.minutes(15), ) create_update_cf_stackset_lambda.node.default_child.cfn_options.metadata = suppress_lambda_policies( ) role_child_nodes = create_update_cf_stackset_lambda.role.node.find_all() role_child_nodes[ 2].node.default_child.cfn_options.metadata = suppress_pipeline_policy( ) # Create codepipeline action create_stackset_action = codepipeline_actions.LambdaInvokeAction( action_name=action_name, inputs=[source_output], variables_namespace=f"{action_name}-namespace", lambda_=create_update_cf_stackset_lambda, user_parameters={ "stackset_name": stack_name, "artifact": artifact, "template_file": template_file, "stage_params_file": stage_params_file, "accound_ids": accound_ids, "org_ids": org_ids, "regions": regions, }, run_order=1, ) return (create_update_cf_stackset_lambda.function_arn, create_stackset_action)
def create_data_baseline_job( scope, # NOSONAR:S107 this function is designed to take many arguments blueprint_bucket, assets_bucket, baseline_job_name, training_data_location, baseline_job_output_location, endpoint_name, instance_type, instance_volume_size, max_runtime_seconds, kms_key_arn, kms_key_arn_provided_condition, stack_name, ): """ create_baseline_job creates a data baseline processing job in a lambda invoked codepipeline action :scope: CDK Construct scope that's needed to create CDK resources :blueprint_bucket: CDK object of the blueprint bucket that contains resources for BYOM pipeline :assets_bucket: the bucket cdk object where pipeline assets are stored :baseline_job_name: name of the baseline job to be created :training_data_location: location of the training data used to train the deployed model :baseline_job_output_location: S3 prefix in the S3 assets bucket to store the output of the job :endpoint_name: name of the deployed SageMaker endpoint to be monitored :instance_type: compute instance type for the baseline job, in the form of a CDK CfnParameter object :instance_volume_size: volume size of the EC2 instance :max_runtime_seconds: max time the job is allowd to run :kms_key_arn: kms key arn to encrypt the baseline job's output :stack_name: model monitor stack name :return: codepipeline action in a form of a CDK object that can be attached to a codepipeline stage """ s3_read = s3_policy_read([ f"arn:aws:s3:::{assets_bucket.bucket_name}", f"arn:aws:s3:::{assets_bucket.bucket_name}/{training_data_location}", ]) s3_write = s3_policy_write([ f"arn:aws:s3:::{baseline_job_output_location}/*", ]) create_baseline_job_policy = sagemaker_baseline_job_policy( baseline_job_name) sagemaker_logs_policy = sagemaker_logs_metrics_policy_document( scope, "BaselineLogsMetrcis") # Kms Key permissions kms_policy = kms_policy_document(scope, "BaselineKmsPolicy", kms_key_arn) # add conditions to KMS and ECR policies core.Aspects.of(kms_policy).add( ConditionalResources(kms_key_arn_provided_condition)) # create sagemaker role sagemaker_role = create_service_role( scope, "create_baseline_sagemaker_role", "sagemaker.amazonaws.com", "Role that is create sagemaker model Lambda function assumes to create a baseline job.", ) # attach the conditional policies kms_policy.attach_to_role(sagemaker_role) # create a trust relation to assume the Role sagemaker_role.add_to_policy( iam.PolicyStatement(actions=["sts:AssumeRole"], resources=[sagemaker_role.role_arn])) # creating a role so that this lambda can create a baseline job lambda_role = create_service_role( scope, "create_baseline_job_lambda_role", lambda_service, "Role that is create_data_baseline_job Lambda function assumes to create a baseline job in the pipeline.", ) sagemaker_logs_policy.attach_to_role(sagemaker_role) sagemaker_role.add_to_policy(create_baseline_job_policy) sagemaker_role.add_to_policy(s3_read) sagemaker_role.add_to_policy(s3_write) sagemaker_role_nodes = sagemaker_role.node.find_all() sagemaker_role_nodes[ 2].node.default_child.cfn_options.metadata = suppress_pipeline_policy( ) lambda_role.add_to_policy( iam.PolicyStatement(actions=["iam:PassRole"], resources=[sagemaker_role.role_arn])) lambda_role.add_to_policy(create_baseline_job_policy) lambda_role.add_to_policy(s3_write) lambda_role.add_to_policy(s3_read) add_logs_policy(lambda_role) # defining the lambda function that gets invoked in this stage create_baseline_job_lambda = lambda_.Function( scope, "create_data_baseline_job", runtime=lambda_.Runtime.PYTHON_3_8, handler=lambda_handler, role=lambda_role, code=lambda_.Code.from_bucket( blueprint_bucket, "blueprints/byom/lambdas/create_data_baseline_job.zip"), environment={ "BASELINE_JOB_NAME": baseline_job_name, "ASSETS_BUCKET": assets_bucket.bucket_name, "SAGEMAKER_ENDPOINT_NAME": endpoint_name, "TRAINING_DATA_LOCATION": training_data_location, "BASELINE_JOB_OUTPUT_LOCATION": baseline_job_output_location, "INSTANCE_TYPE": instance_type, "INSTANCE_VOLUME_SIZE": instance_volume_size, "MAX_RUNTIME_SECONDS": max_runtime_seconds, "ROLE_ARN": sagemaker_role.role_arn, "KMS_KEY_ARN": kms_key_arn, "STACK_NAME": stack_name, "LOG_LEVEL": "INFO", }, timeout=core.Duration.minutes(10), ) create_baseline_job_lambda.node.default_child.cfn_options.metadata = suppress_lambda_policies( ) role_child_nodes = create_baseline_job_lambda.role.node.find_all() role_child_nodes[ 2].node.default_child.cfn_options.metadata = suppress_pipeline_policy( ) return create_baseline_job_lambda