def create_monitoring_schedule(
scope,
blueprint_bucket,
assets_bucket,
baseline_job_output_location,
baseline_job_name,
monitoring_schedual_name,
monitoring_output_location,
schedule_expression,
endpoint_name,
instance_type,
instance_volume_size,
max_runtime_seconds,
monitoring_type,
stack_name,
):
"""
create_monitoring_schedule creates a model monitoring job in a lambda invoked codepipeline action
:scope: CDK Construct scope that's needed to create CDK resources
:blueprint_bucket: CDK object of the blueprint bucket that contains resources for BYOM pipeline
:assets_bucket: the bucket cdk object where pipeline assets are stored
:baseline_job_output_location: S3 prefix in the S3 assets bucket to store the output of the job
:baseline_job_name: name of the baseline job
:monitoring_schedual_name: name of the monitoring job to be created
:schedule_expression cron job expression
:endpoint_name: name of the deployed SageMaker endpoint to be monitored
:instance_type: compute instance type for the baseline job, in the form of a CDK CfnParameter object
:instance_volume_size: volume size of the EC2 instance
:monitoring_type: type of monitoring to be created
:max_runtime_seconds: max time the job is allowd to run
:stack_name: name of the model monitoring satck
:return: codepipeline action in a form of a CDK object that can be attached to a codepipeline stage
"""
create_monitoring_schedule_policy = iam.PolicyStatement(
actions=[
"sagemaker:DescribeEndpointConfig",
"sagemaker:DescribeEndpoint",
"sagemaker:CreateMonitoringSchedule",
"sagemaker:DescribeMonitoringSchedule",
"sagemaker:StopMonitoringSchedule",
"sagemaker:DeleteMonitoringSchedule",
"sagemaker:DescribeProcessingJob",
],
resources=[
(f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:"
f"endpoint/{endpoint_name.value_as_string}*"),
(f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:"
f"endpoint-config/{endpoint_name.value_as_string}*"),
(f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:"
f"monitoring-schedule/{monitoring_schedual_name.value_as_string}"
),
(f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:"
f"processing-job/{baseline_job_name.value_as_string}"),
],
)
s3_policy = iam.PolicyStatement(
actions=[
"s3:ListBucket",
"s3:GetObject",
"s3:PutObject",
],
resources=[
assets_bucket.bucket_arn,
assets_bucket.arn_for_objects("*"),
],
)
sagemaker_logs_policy = iam.PolicyStatement(
actions=[
"cloudwatch:PutMetricData",
"logs:CreateLogStream",
"logs:PutLogEvents",
"logs:CreateLogGroup",
"logs:DescribeLogStreams",
],
resources=["*"],
)
# create sagemaker role
sagemaker_role = iam.Role(
scope,
"create_monitoring_scheduale_sagemaker_role",
assumed_by=iam.ServicePrincipal("sagemaker.amazonaws.com"),
description=
"Role that is create sagemaker model Lambda function assumes to create a model in the pipeline.",
)
# create a trust relation to assume the Role
sagemaker_role.add_to_policy(
iam.PolicyStatement(actions=["sts:AssumeRole"],
resources=[sagemaker_role.role_arn]))
# creating a role so that this lambda can create a baseline job
lambda_role = iam.Role(
scope,
"create_monitoring_scheduale_role",
assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
description=
"Role that is create_data_baseline_job Lambda function assumes to create a baseline job in the pipeline.",
)
sagemaker_role.add_to_policy(create_monitoring_schedule_policy)
sagemaker_role.add_to_policy(sagemaker_logs_policy)
sagemaker_role.add_to_policy(s3_policy)
sagemaker_role_nodes = sagemaker_role.node.find_all()
sagemaker_role_nodes[
2].node.default_child.cfn_options.metadata = suppress_pipeline_policy(
)
lambda_role.add_to_policy(
iam.PolicyStatement(actions=["iam:PassRole"],
resources=[sagemaker_role.role_arn]))
lambda_role.add_to_policy(create_monitoring_schedule_policy)
lambda_role.add_to_policy(s3_policy)
add_logs_policy(lambda_role)
# defining the lambda function that gets invoked in this stage
create_moniroring_schedule_lambda = lambda_.Function(
scope,
"create_moniroring_schedule",
runtime=lambda_.Runtime.PYTHON_3_8,
handler="main.handler",
role=lambda_role,
code=lambda_.Code.from_bucket(
blueprint_bucket,
"blueprints/byom/lambdas/create_model_monitoring_schedule.zip"),
environment={
"BASELINE_JOB_NAME": baseline_job_name.value_as_string,
"BASELINE_JOB_OUTPUT_LOCATION":
baseline_job_output_location.value_as_string,
"ASSETS_BUCKET": assets_bucket.bucket_name,
"SAGEMAKER_ENDPOINT_NAME": f"{endpoint_name.value_as_string}",
"MONITORING_SCHEDULE_NAME":
monitoring_schedual_name.value_as_string,
"MONITORING_OUTPUT_LOCATION":
monitoring_output_location.value_as_string,
"SCHEDULE_EXPRESSION": schedule_expression.value_as_string,
"INSTANCE_TYPE": instance_type.value_as_string,
"INSTANCE_VOLUME_SIZE": instance_volume_size.value_as_string,
"MAX_RUNTIME_SECONDS": max_runtime_seconds.value_as_string,
"ROLE_ARN": sagemaker_role.role_arn,
"MONITORING_TYPE": monitoring_type.value_as_string,
"STACK_NAME": stack_name,
"LOG_LEVEL": "INFO",
},
timeout=core.Duration.minutes(10),
)
create_moniroring_schedule_lambda.node.default_child.cfn_options.metadata = suppress_cloudwatch_policy(
)
role_child_nodes = create_moniroring_schedule_lambda.role.node.find_all()
role_child_nodes[
2].node.default_child.cfn_options.metadata = suppress_pipeline_policy(
)
# Create codepipeline action
create_moniroring_schedule_action = codepipeline_actions.LambdaInvokeAction(
action_name="create_monitoring_schedule",
inputs=[],
outputs=[],
variables_namespace="monitoring_schedule",
lambda_=create_moniroring_schedule_lambda,
run_order=2, # this runs second in the deploy stage
)
return (create_moniroring_schedule_lambda.function_arn,
create_moniroring_schedule_action)
def batch_transform(
scope,
blueprint_bucket,
assets_bucket,
model_name,
inference_instance,
batch_inference_data,
sm_layer,
):
"""
batch_transform creates a sagemaker batch transform job in a lambda invoked codepipeline action
:scope: CDK Construct scope that's needed to create CDK resources
:blueprint_bucket: CDK object of the blueprint bucket that contains resources for BYOM pipeline
:assets_bucket: the bucket cdk object where pipeline assets are stored
:model_name: name of the sagemaker model to be created, in the form of a CDK CfnParameter object
:inference_instance: compute instance type for the sagemaker inference endpoint, in the form of
a CDK CfnParameter object
:batch_inference_data: location of the batch inference data in assets bucket, in the form of
a CDK CfnParameter object
:is_batch_transform: a CDK CfnCondition object that says if inference type is batch transform or not
:sm_layer: sagemaker lambda layer
:return: codepipeline action in a form of a CDK object that can be attached to a codepipeline stage
"""
batch_transform_policy = iam.PolicyStatement(
actions=[
"sagemaker:CreateTransformJob",
"s3:ListBucket",
"s3:GetObject",
"s3:PutObject",
],
resources=[
assets_bucket.bucket_arn,
assets_bucket.arn_for_objects("*"),
(f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:"
f"transform-job/{model_name.value_as_string}-*"),
],
)
lambda_role = iam.Role(
scope,
"batch_transform_lambda_role",
assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
description=
("Role that creates a lambda function assumes to create a sagemaker batch transform "
"job in the aws mlops pipeline."),
)
lambda_role.add_to_policy(batch_transform_policy)
lambda_role.add_to_policy(codepipeline_policy())
add_logs_policy(lambda_role)
# defining batch transform lambda function
batch_transform = lambda_.Function(
scope,
"batch_transform",
runtime=lambda_.Runtime.PYTHON_3_8,
handler="main.handler",
layers=[sm_layer],
role=lambda_role,
code=lambda_.Code.from_bucket(
blueprint_bucket, "blueprints/byom/lambdas/batch_transform.zip"),
environment={
"model_name": model_name.value_as_string,
"inference_instance": inference_instance.value_as_string,
"assets_bucket": assets_bucket.bucket_name,
"batch_inference_data": batch_inference_data.value_as_string,
"LOG_LEVEL": "INFO",
},
)
batch_transform.node.default_child.cfn_options.metadata = suppress_cloudwatch_policy(
)
role_child_nodes = batch_transform.role.node.find_all()
role_child_nodes[
2].node.default_child.cfn_options.metadata = suppress_pipeline_policy(
)
batch_transform_action = codepipeline_actions.LambdaInvokeAction(
action_name="batch_transform",
inputs=[],
outputs=[],
variables_namespace="batch_transform",
lambda_=batch_transform,
run_order=2, # this runs second in the deploy stage
)
return (batch_transform.function_arn, batch_transform_action)
def create_model(
scope,
blueprint_bucket,
assets_bucket,
model_name,
model_artifact_location,
custom_container,
model_framework,
model_framework_version,
container_uri,
sm_layer,
):
"""
create_model creates a sagemaker model in a lambda invoked codepipeline action
:scope: CDK Construct scope that's needed to create CDK resources
:blueprint_bucket: CDK object of the blueprint bucket that contains resources for BYOM pipeline
:assets_bucket: the bucket cdk object where pipeline assets are stored
:model_name: name of the sagemaker model to be created, in the form of a CDK CfnParameter object
:model_artifact_location: path to the model artifact in the S3 bucket: assets_bucket
:custom_container: whether to the model is a custom algorithm or a sagemaker algorithmm, in the form of
a CDK CfnParameter object
:model_framework: name of the framework if the model is a sagemaker algorithm, in the form of
a CDK CfnParameter object
:model_framework_version: version of the framework if the model is a sagemaker algorithm, in the form of
a CDK CfnParameter object
:container_uri: URI for the container registry that stores the model if the model is a custom algorithm
:sm_layer: sagemaker lambda layer
:return: codepipeline action in a form of a CDK object that can be attached to a codepipeline stage
"""
create_model_policy = iam.PolicyStatement(
actions=[
"sagemaker:CreateModel",
"sagemaker:DescribeModel",
"sagemaker:DeleteModel",
],
resources=[
# Lambda that uses this polict requires access to all objects in the assets bucket
f"arn:{core.Aws.PARTITION}:s3:::{assets_bucket.bucket_name}/*",
(f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}"
f":model/{model_name.value_as_string}"),
],
)
s3_policy = iam.PolicyStatement(
actions=[
"s3:GetObject",
"s3:PutObject",
"s3:ListBucket",
],
resources=[
assets_bucket.arn_for_objects("*"), assets_bucket.bucket_arn
],
)
# creating this policy for sagemaker create endpoint in custom model
ecr_policy = iam.PolicyStatement(
actions=[
"ecr:BatchGetImage",
"ecr:BatchCheckLayerAvailability",
"ecr:DescribeImages",
"ecr:DescribeRepositories",
"ecr:GetDownloadUrlForLayer",
],
resources=[
f"arn:{core.Aws.PARTITION}:ecr:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:repository/awsmlopsmodels*"
],
)
ecr_token_policy = iam.PolicyStatement(
actions=["ecr:GetAuthorizationToken"],
resources=[
"*"
], # GetAuthorizationToken can not be bound to resources other than *
)
# creating a role for the lambda function so that it can create a model in sagemaker
sagemaker_role = iam.Role(
scope,
"create_model_sagemaker_role",
assumed_by=iam.ServicePrincipal("sagemaker.amazonaws.com"),
description=
"Role that is create sagemaker model Lambda function assumes to create a model in the pipeline.",
)
lambda_role = iam.Role(
scope,
"create_model_lambda_role",
assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
description=
"Role that is create sagemaker model Lambda function assumes to create a model in the pipeline.",
)
sagemaker_role.add_to_policy(create_model_policy)
sagemaker_role.add_to_policy(s3_policy)
sagemaker_role.add_to_policy(ecr_policy)
sagemaker_role.add_to_policy(ecr_token_policy)
sagemaker_role_nodes = sagemaker_role.node.find_all()
sagemaker_role_nodes[
2].node.default_child.cfn_options.metadata = suppress_ecr_policy()
lambda_role.add_to_policy(
iam.PolicyStatement(actions=["iam:PassRole"],
resources=[sagemaker_role.role_arn]))
lambda_role.add_to_policy(create_model_policy)
lambda_role.add_to_policy(s3_policy)
add_logs_policy(lambda_role)
# defining the lambda function that gets invoked by codepipeline in this step
create_sagemaker_model = lambda_.Function(
scope,
"create_sagemaker_model",
runtime=lambda_.Runtime.PYTHON_3_8,
handler="main.handler",
timeout=core.Duration.seconds(60),
code=lambda_.Code.from_bucket(
blueprint_bucket,
"blueprints/byom/lambdas/create_sagemaker_model.zip"),
layers=[sm_layer],
role=lambda_role,
environment={
"custom_container":
custom_container.value_as_string,
"model_framework":
model_framework.value_as_string,
"model_framework_version":
model_framework_version.value_as_string,
"model_name":
model_name.value_as_string,
"model_artifact_location":
assets_bucket.s3_url_for_object(
model_artifact_location.value_as_string),
"create_model_role_arn":
sagemaker_role.role_arn,
"container_uri":
container_uri,
"LOG_LEVEL":
"INFO",
},
)
create_sagemaker_model.node.default_child.cfn_options.metadata = suppress_cloudwatch_policy(
)
role_child_nodes = create_sagemaker_model.role.node.find_all()
role_child_nodes[
2].node.default_child.cfn_options.metadata = suppress_pipeline_policy(
)
# creating the codepipeline action that invokes create model lambda
create_sagemaker_model_action = codepipeline_actions.LambdaInvokeAction(
action_name="create_sagemaker_model",
inputs=[],
outputs=[],
lambda_=create_sagemaker_model,
run_order=1, # runs first in the Deploy stage
)
return (create_sagemaker_model.function_arn, create_sagemaker_model_action)
def create_endpoint(scope, blueprint_bucket, assets_bucket, model_name,
inference_instance):
"""
create_endpoint creates a sagemaker inference endpoint in a lambda invoked codepipeline action
:scope: CDK Construct scope that's needed to create CDK resources
:blueprint_bucket: CDK object of the blueprint bucket that contains resources for BYOM pipeline
:assets_bucket: the bucket cdk object where pipeline assets are stored
:model_name: name of the sagemaker model to be created, in the form of a CDK CfnParameter object
:inference_instance: compute instance type for the sagemaker inference endpoint, in the form of
a CDK CfnParameter object
:is_realtime_inference: a CDK CfnCondition object that says if inference type is realtime or not
:return: codepipeline action in a form of a CDK object that can be attached to a codepipeline stage
"""
create_endpoint_policy = iam.PolicyStatement(
actions=[
"sagemaker:CreateEndpoint",
"sagemaker:CreateEndpointConfig",
"sagemaker:DeleteEndpointConfig",
"sagemaker:DescribeEndpointConfig",
"sagemaker:DescribeEndpoint",
],
resources=[
(f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:"
f"endpoint/{model_name.value_as_string}-endpoint"),
(f"arn:{core.Aws.PARTITION}:sagemaker:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:"
f"endpoint-config/{model_name.value_as_string}-endpoint-config"),
],
)
# creating a role so that this lambda can create a sagemaker endpoint and endpoint config
lambda_role = iam.Role(
scope,
"create_endpoint_lambda_role",
assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"),
description=
"Role that is create sagemaker model Lambda function assumes to create a model in the pipeline.",
)
lambda_role.add_to_policy(create_endpoint_policy)
add_logs_policy(lambda_role)
# defining the lambda function that gets invoked in this stage
create_sagemaker_endpoint = lambda_.Function(
scope,
"create_sagemaker_endpoint",
runtime=lambda_.Runtime.PYTHON_3_8,
handler="main.handler",
role=lambda_role,
code=lambda_.Code.from_bucket(
blueprint_bucket,
"blueprints/byom/lambdas/create_sagemaker_endpoint.zip"),
environment={
"model_name": model_name.value_as_string,
"inference_instance": inference_instance.value_as_string,
"assets_bucket": assets_bucket.bucket_name,
"LOG_LEVEL": "INFO",
},
timeout=core.Duration.minutes(10),
)
create_sagemaker_endpoint.node.default_child.cfn_options.metadata = suppress_cloudwatch_policy(
)
role_child_nodes = create_sagemaker_endpoint.role.node.find_all()
role_child_nodes[
2].node.default_child.cfn_options.metadata = suppress_pipeline_policy(
)
# create_endpoint_action = core.Fn.condition_if("isRealtimeInference",
create_endpoint_action = codepipeline_actions.LambdaInvokeAction(
action_name="create_sagemaker_endpoint",
inputs=[],
outputs=[],
variables_namespace="sagemaker_endpoint",
lambda_=create_sagemaker_endpoint,
run_order=2, # this runs second in the deploy stage
)
return (create_sagemaker_endpoint.function_arn, create_endpoint_action)
def build_action(scope, source_output):
"""
build_action configures a codepipeline action that takes Dockerfile and creates a container image
:scope: CDK Construct scope that's needed to create CDK resources
:source_output: output of the source stage in codepipeline
:is_custom_container: a CDK CfnCondition object, if true, it creates resources for the pipeline action
:return: codepipeline action in a form of a CDK object that can be attached to a codepipeline stage
"""
model_containers = ecr.Repository(scope, "awsmlopsmodels")
codebuild_role = iam.Role(
scope,
"codebuildRole",
assumed_by=iam.ServicePrincipal("codebuild.amazonaws.com"))
codebuild_policy = iam.PolicyStatement(actions=[
"ecr:BatchCheckLayerAvailability",
"ecr:CompleteLayerUpload",
"ecr:InitiateLayerUpload",
"ecr:PutImage",
"ecr:UploadLayerPart",
],
resources=[
model_containers.repository_arn,
])
codebuild_role.add_to_policy(codebuild_policy)
codebuild_role.add_to_policy(
iam.PolicyStatement(actions=["ecr:GetAuthorizationToken"],
resources=["*"]))
codebuild_role_child_nodes = codebuild_role.node.find_all()
codebuild_role_child_nodes[
3].cfn_options.metadata = suppress_pipeline_policy()
# codebuild setup for build stage
container_factory_project = codebuild.PipelineProject(
scope,
"Container_Factory",
build_spec=codebuild.BuildSpec.from_object({
"version": "0.2",
"phases": {
"pre_build": {
"commands": [
"echo Logging in to Amazon ECR...",
"aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com",
"find . -iname \"serve\" -exec chmod 777 \"{}\" \\;",
"find . -iname \"train\" -exec chmod 777 \"{}\" \\;",
]
},
"build": {
"commands": [
"echo Build started on `date`",
"echo Building the Docker image...",
"docker build -t $IMAGE_REPO_NAME:$IMAGE_TAG .",
"docker tag $IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG"
]
},
"post_build": {
"commands": [
"echo Build completed on `date`",
"echo Pushing the Docker image...",
"docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG"
]
}
},
}),
environment=codebuild.BuildEnvironment(
build_image=codebuild.LinuxBuildImage.STANDARD_4_0,
compute_type=codebuild.ComputeType.SMALL,
environment_variables={
"AWS_DEFAULT_REGION": {
"value": core.Aws.REGION
},
"AWS_ACCOUNT_ID": {
"value": core.Aws.ACCOUNT_ID
},
"IMAGE_REPO_NAME": {
"value": model_containers.repository_name
},
"IMAGE_TAG": {
"value": "latest"
}
},
privileged=True),
role=codebuild_role)
build_action_definition = codepipeline_actions.CodeBuildAction(
action_name="CodeBuild",
project=container_factory_project,
input=source_output,
outputs=[codepipeline.Artifact()])
container_uri = f'{core.Aws.ACCOUNT_ID}.dkr.ecr.{core.Aws.REGION}.amazonaws.com/{model_containers.repository_name}:latest'
return build_action_definition, container_uri
def build_action(scope, ecr_repository_name, image_tag, source_output):
"""
build_action configures a codepipeline action with repository name and tag
:scope: CDK Construct scope that's needed to create CDK resources
:ecr_repository_name: name of Amazon ECR repository where the image will be stored
:image_tag: docker image tag to be assigned.
:return: codepipeline action in a form of a CDK object that can be attached to a codepipeline stage
"""
codebuild_role = iam.Role(
scope,
"codebuildRole",
assumed_by=iam.ServicePrincipal("codebuild.amazonaws.com"))
codebuild_policy = iam.PolicyStatement(
actions=[
"ecr:BatchCheckLayerAvailability",
"ecr:CompleteLayerUpload",
"ecr:InitiateLayerUpload",
"ecr:PutImage",
"ecr:UploadLayerPart",
],
resources=[
f"arn:{core.Aws.PARTITION}:ecr:{core.Aws.REGION}:{core.Aws.ACCOUNT_ID}:repository/{ecr_repository_name}",
],
)
codebuild_role.add_to_policy(codebuild_policy)
codebuild_role.add_to_policy(
iam.PolicyStatement(actions=["ecr:GetAuthorizationToken"],
resources=["*"]))
codebuild_role_child_nodes = codebuild_role.node.find_all()
codebuild_role_child_nodes[
3].cfn_options.metadata = suppress_pipeline_policy()
# codebuild setup for build stage
container_factory_project = codebuild.PipelineProject(
scope,
"Container_Factory",
build_spec=codebuild.BuildSpec.from_object({
"version": "0.2",
"phases": {
"pre_build": {
"commands": [
"echo Logging in to Amazon ECR...",
("aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS "
"--password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com"
),
'find . -iname "serve" -exec chmod 777 "{}" \\;',
'find . -iname "train" -exec chmod 777 "{}" \\;',
]
},
"build": {
"commands": [
"echo Build started on `date`",
"echo Building the Docker image...",
"docker build -t $IMAGE_REPO_NAME:$IMAGE_TAG .",
("docker tag $IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_ID.dkr.ecr."
"$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG"
),
]
},
"post_build": {
"commands": [
"echo Build completed on `date`",
"echo Pushing the Docker image...",
("docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/"
"$IMAGE_REPO_NAME:$IMAGE_TAG"),
]
},
},
}),
environment=codebuild.BuildEnvironment(
build_image=codebuild.LinuxBuildImage.STANDARD_4_0,
compute_type=codebuild.ComputeType.SMALL,
environment_variables={
"AWS_DEFAULT_REGION": {
"value": core.Aws.REGION
},
"AWS_ACCOUNT_ID": {
"value": core.Aws.ACCOUNT_ID
},
"IMAGE_REPO_NAME": {
"value": ecr_repository_name
},
"IMAGE_TAG": {
"value": image_tag
},
},
privileged=True,
),
role=codebuild_role,
)
build_action_definition = codepipeline_actions.CodeBuildAction(
action_name="CodeBuild",
project=container_factory_project,
input=source_output,
outputs=[codepipeline.Artifact()],
)
container_uri = f"{core.Aws.ACCOUNT_ID}.dkr.ecr.{core.Aws.REGION}.amazonaws.com/{ecr_repository_name}:{image_tag}"
return build_action_definition, container_uri
def create_stackset_action(
scope, # NOSONAR:S107 this function is designed to take many arguments
action_name,
blueprint_bucket,
source_output,
artifact,
template_file,
stage_params_file,
accound_ids,
org_ids,
regions,
assets_bucket,
stack_name,
):
"""
create_stackset_action an invokeLambda action to be added to AWS Codepipeline stage
:scope: CDK Construct scope that's needed to create CDK resources
:action_name: name of the StackSet action
:blueprint_bucket: CDK object of the blueprint bucket that contains resources for BYOM pipeline
:source_output: CDK object of the Source action's output
:artifact: name of the input aritifcat to the StackSet action
:template_file: name of the Cloudformation template to be deployed
:stage_params_file: name of the template parameters for the satge
:accound_ids: list of AWS acounts where the stack with be deployed
:org_ids: list of AWS orginizational ids where the stack with be deployed
:regions: list of regions where the stack with be deployed
:assets_bucket: the bucket cdk object where pipeline assets are stored
:stack_name: name of the stack to be deployed
:return: codepipeline invokeLambda action in a form of a CDK object that can be attached to a codepipeline stage
"""
# creating a role so that this lambda can create a baseline job
lambda_role = create_service_role(
scope,
f"{action_name}_role",
lambda_service,
"The role that is assumed by create_update_cf_stackset Lambda function.",
)
# make the stackset name unique
stack_name = f"{stack_name}-{str(uuid.uuid4())[:8]}"
# cloudformation stackset permissions
cloudformation_stackset_permissions = cloudformation_stackset_policy(
stack_name)
cloudformation_stackset_instances_permissions = cloudformation_stackset_instances_policy(
stack_name)
lambda_role.add_to_policy(cloudformation_stackset_permissions)
lambda_role.add_to_policy(cloudformation_stackset_instances_permissions)
add_logs_policy(lambda_role)
# defining the lambda function that gets invoked in this stage
create_update_cf_stackset_lambda = lambda_.Function(
scope,
f"{action_name}_stackset_lambda",
runtime=lambda_.Runtime.PYTHON_3_8,
handler="main.lambda_handler",
role=lambda_role,
code=lambda_.Code.from_bucket(
blueprint_bucket,
"blueprints/byom/lambdas/create_update_cf_stackset.zip"),
timeout=core.Duration.minutes(15),
)
create_update_cf_stackset_lambda.node.default_child.cfn_options.metadata = suppress_lambda_policies(
)
role_child_nodes = create_update_cf_stackset_lambda.role.node.find_all()
role_child_nodes[
2].node.default_child.cfn_options.metadata = suppress_pipeline_policy(
)
# Create codepipeline action
create_stackset_action = codepipeline_actions.LambdaInvokeAction(
action_name=action_name,
inputs=[source_output],
variables_namespace=f"{action_name}-namespace",
lambda_=create_update_cf_stackset_lambda,
user_parameters={
"stackset_name": stack_name,
"artifact": artifact,
"template_file": template_file,
"stage_params_file": stage_params_file,
"accound_ids": accound_ids,
"org_ids": org_ids,
"regions": regions,
},
run_order=1,
)
return (create_update_cf_stackset_lambda.function_arn,
create_stackset_action)
def create_data_baseline_job(
scope, # NOSONAR:S107 this function is designed to take many arguments
blueprint_bucket,
assets_bucket,
baseline_job_name,
training_data_location,
baseline_job_output_location,
endpoint_name,
instance_type,
instance_volume_size,
max_runtime_seconds,
kms_key_arn,
kms_key_arn_provided_condition,
stack_name,
):
"""
create_baseline_job creates a data baseline processing job in a lambda invoked codepipeline action
:scope: CDK Construct scope that's needed to create CDK resources
:blueprint_bucket: CDK object of the blueprint bucket that contains resources for BYOM pipeline
:assets_bucket: the bucket cdk object where pipeline assets are stored
:baseline_job_name: name of the baseline job to be created
:training_data_location: location of the training data used to train the deployed model
:baseline_job_output_location: S3 prefix in the S3 assets bucket to store the output of the job
:endpoint_name: name of the deployed SageMaker endpoint to be monitored
:instance_type: compute instance type for the baseline job, in the form of a CDK CfnParameter object
:instance_volume_size: volume size of the EC2 instance
:max_runtime_seconds: max time the job is allowd to run
:kms_key_arn: kms key arn to encrypt the baseline job's output
:stack_name: model monitor stack name
:return: codepipeline action in a form of a CDK object that can be attached to a codepipeline stage
"""
s3_read = s3_policy_read([
f"arn:aws:s3:::{assets_bucket.bucket_name}",
f"arn:aws:s3:::{assets_bucket.bucket_name}/{training_data_location}",
])
s3_write = s3_policy_write([
f"arn:aws:s3:::{baseline_job_output_location}/*",
])
create_baseline_job_policy = sagemaker_baseline_job_policy(
baseline_job_name)
sagemaker_logs_policy = sagemaker_logs_metrics_policy_document(
scope, "BaselineLogsMetrcis")
# Kms Key permissions
kms_policy = kms_policy_document(scope, "BaselineKmsPolicy", kms_key_arn)
# add conditions to KMS and ECR policies
core.Aspects.of(kms_policy).add(
ConditionalResources(kms_key_arn_provided_condition))
# create sagemaker role
sagemaker_role = create_service_role(
scope,
"create_baseline_sagemaker_role",
"sagemaker.amazonaws.com",
"Role that is create sagemaker model Lambda function assumes to create a baseline job.",
)
# attach the conditional policies
kms_policy.attach_to_role(sagemaker_role)
# create a trust relation to assume the Role
sagemaker_role.add_to_policy(
iam.PolicyStatement(actions=["sts:AssumeRole"],
resources=[sagemaker_role.role_arn]))
# creating a role so that this lambda can create a baseline job
lambda_role = create_service_role(
scope,
"create_baseline_job_lambda_role",
lambda_service,
"Role that is create_data_baseline_job Lambda function assumes to create a baseline job in the pipeline.",
)
sagemaker_logs_policy.attach_to_role(sagemaker_role)
sagemaker_role.add_to_policy(create_baseline_job_policy)
sagemaker_role.add_to_policy(s3_read)
sagemaker_role.add_to_policy(s3_write)
sagemaker_role_nodes = sagemaker_role.node.find_all()
sagemaker_role_nodes[
2].node.default_child.cfn_options.metadata = suppress_pipeline_policy(
)
lambda_role.add_to_policy(
iam.PolicyStatement(actions=["iam:PassRole"],
resources=[sagemaker_role.role_arn]))
lambda_role.add_to_policy(create_baseline_job_policy)
lambda_role.add_to_policy(s3_write)
lambda_role.add_to_policy(s3_read)
add_logs_policy(lambda_role)
# defining the lambda function that gets invoked in this stage
create_baseline_job_lambda = lambda_.Function(
scope,
"create_data_baseline_job",
runtime=lambda_.Runtime.PYTHON_3_8,
handler=lambda_handler,
role=lambda_role,
code=lambda_.Code.from_bucket(
blueprint_bucket,
"blueprints/byom/lambdas/create_data_baseline_job.zip"),
environment={
"BASELINE_JOB_NAME": baseline_job_name,
"ASSETS_BUCKET": assets_bucket.bucket_name,
"SAGEMAKER_ENDPOINT_NAME": endpoint_name,
"TRAINING_DATA_LOCATION": training_data_location,
"BASELINE_JOB_OUTPUT_LOCATION": baseline_job_output_location,
"INSTANCE_TYPE": instance_type,
"INSTANCE_VOLUME_SIZE": instance_volume_size,
"MAX_RUNTIME_SECONDS": max_runtime_seconds,
"ROLE_ARN": sagemaker_role.role_arn,
"KMS_KEY_ARN": kms_key_arn,
"STACK_NAME": stack_name,
"LOG_LEVEL": "INFO",
},
timeout=core.Duration.minutes(10),
)
create_baseline_job_lambda.node.default_child.cfn_options.metadata = suppress_lambda_policies(
)
role_child_nodes = create_baseline_job_lambda.role.node.find_all()
role_child_nodes[
2].node.default_child.cfn_options.metadata = suppress_pipeline_policy(
)
return create_baseline_job_lambda