def test_handler_success(
sm_describe_endpoint_config_params, sm_create_endpoint_config_params,
sm_create_endpoint_config_response, cp_expected_params,
sm_describe_endpoint_params, sm_create_endpoint_params,
sm_create_endpoint_response, sm_describe_endpoint_response_2, event):
sm_client = get_client("sagemaker")
cp_client = get_client("codepipeline")
sm_stubber = Stubber(sm_client)
cp_stubber = Stubber(cp_client)
# endpoint config creation
sm_describe_endpoint_config_response = {}
cp_response = {}
sm_stubber.add_client_error(
"describe_endpoint_config",
service_error_code="EndpointConfigExists",
service_message="Could not find endpoint configuration",
http_status_code=400,
expected_params=sm_describe_endpoint_config_params,
)
sm_stubber.add_response(
"create_endpoint_config",
sm_create_endpoint_config_response,
sm_create_endpoint_config_params,
)
# endpoint creation
sm_stubber.add_client_error(
"describe_endpoint",
service_error_code="EndpointExists",
service_message="Could not find endpoint",
http_status_code=400,
expected_params=sm_describe_endpoint_params,
)
sm_stubber.add_response("create_endpoint", sm_create_endpoint_response,
sm_create_endpoint_params)
sm_stubber.add_response(
"describe_endpoint",
sm_describe_endpoint_response_2,
sm_describe_endpoint_params,
)
cp_stubber.add_response("put_job_success_result", cp_response,
cp_expected_params)
expected_log_message = (
"Sent success message back to codepipeline with job_id: test_job_id")
with sm_stubber:
with cp_stubber:
handler(event, {})
cp_stubber.assert_no_pending_responses()
reset_client()
def test_handler_success(
sm_create_monitoring_expected_params,
sm_create_monitoring_response_200,
sm_describe_monitoring_schedule_response,
cp_expected_params_success,
sm_describe_monitoring_scheduale_params,
event,
):
sm_client = get_client("sagemaker")
cp_client = get_client("codepipeline")
sm_stubber = Stubber(sm_client)
cp_stubber = Stubber(cp_client)
cp_response = {}
# job creation
sm_stubber.add_client_error(
"describe_monitoring_schedule",
service_error_code="MonitorJobExists",
service_message="Could not find requested job with name",
http_status_code=400,
expected_params=sm_describe_monitoring_scheduale_params,
)
# success path
sm_stubber.add_response("create_monitoring_schedule",
sm_create_monitoring_response_200,
sm_create_monitoring_expected_params)
sm_stubber.add_response(
"describe_monitoring_schedule",
sm_describe_monitoring_schedule_response,
sm_describe_monitoring_scheduale_params,
)
cp_stubber.add_response("put_job_success_result", cp_response,
cp_expected_params_success)
with sm_stubber:
with cp_stubber:
handler(event, {})
cp_stubber.assert_no_pending_responses()
reset_client()
def test_handler_failure(lm_expected_params, lm_response_500,
cp_expected_params_failure, event):
lm_client = get_client("lambda")
lm_stubber = Stubber(lm_client)
cp_client = get_client("codepipeline")
cp_stubber = Stubber(cp_client)
cp_response = {}
lm_stubber.add_response("update_function_configuration", lm_response_500,
lm_expected_params)
cp_stubber.add_response("put_job_failure_result", cp_response,
cp_expected_params_failure)
with lm_stubber:
with cp_stubber:
handler(event, {})
cp_stubber.assert_no_pending_responses()
reset_client()
def test_handler_fail(sm_expected_params, sm_response_500,
cp_expected_params_failure, event):
sm_client = get_client("sagemaker")
cp_client = get_client("codepipeline")
sm_stubber = Stubber(sm_client)
cp_stubber = Stubber(cp_client)
cp_response = {}
# fail path
sm_stubber.add_response("create_transform_job", sm_response_500,
sm_expected_params)
cp_stubber.add_response("put_job_failure_result", cp_response,
cp_expected_params_failure)
with sm_stubber:
with cp_stubber:
handler(event, {})
cp_stubber.assert_no_pending_responses()
reset_client()
def test_handler_success(sm_expected_params, sm_response_200, event):
sm_client = get_client("sagemaker")
sm_stubber = Stubber(sm_client)
# success path
sm_stubber.add_response("create_transform_job", sm_response_200,
sm_expected_params)
with sm_stubber:
handler(event, {})
reset_client()
def test_handler_fail(sm_expected_params, sm_response_500, event):
sm_client = get_client("sagemaker")
sm_stubber = Stubber(sm_client)
# fail path
sm_stubber.add_response("create_transform_job", sm_response_500,
sm_expected_params)
with pytest.raises(Exception):
handler(event, {})
reset_client()
def test_handler_success(
sm_describe_model_expected_params,
sm_describe_model_response,
sm_delete_model_expected_params,
sm_create_model_expected_params,
sm_create_model_response,
cp_expected_params,
event,
):
sm_client = get_client("sagemaker")
cp_client = get_client("codepipeline")
sm_stubber = Stubber(sm_client)
cp_stubber = Stubber(cp_client)
# describe model
sm_stubber.add_response("describe_model", sm_describe_model_response, sm_describe_model_expected_params)
# delete model
sm_delete_model_response = {}
sm_stubber.add_response("delete_model", sm_delete_model_response, sm_delete_model_expected_params)
# create model
sm_stubber.add_response("create_model", sm_create_model_response, sm_create_model_expected_params)
# codepipeline
cp_response = {}
cp_stubber.add_response("put_job_success_result", cp_response, cp_expected_params)
with sm_stubber:
with cp_stubber:
handler(event, {})
cp_stubber.assert_no_pending_responses()
reset_client()
def test_handler_success(
sm_create_baseline_expected_params,
sm_create_job_response_200,
event,
):
sm_client = get_client("sagemaker")
sm_stubber = Stubber(sm_client)
# success path
sm_stubber.add_response("create_processing_job", sm_create_job_response_200, sm_create_baseline_expected_params)
with sm_stubber:
handler(event, {})
reset_client()
def wrapper(event, context):
try:
return f(event, context)
except Exception as e:
codepipeline = get_client("codepipeline")
exc_type, exc_value, exc_tb = sys.exc_info()
logger.error(
traceback.format_exception(exc_type, exc_value, exc_tb))
codepipeline.put_job_failure_result(
jobId=event["CodePipeline.job"]["id"],
failureDetails={
"message":
f"Job failed. {str(e)}. Check the logs for more info.",
"type": "JobFailed",
},
)
# with the License. A copy of the License is located at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES #
# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions #
# and limitations under the License. #
# #####################################################################################################################
import os
import botocore
import boto3
from shared.logger import get_logger
from shared.helper import get_client, get_built_in_model_monitor_container_uri
logger = get_logger(__name__)
sm_client = get_client("sagemaker")
def handler(event, context):
baseline_job_name = os.environ["BASELINE_JOB_NAME"]
assets_bucket = os.environ["ASSETS_BUCKET"]
training_data_location = os.environ["TRAINING_DATA_LOCATION"]
baseline_job_output_location = os.environ["BASELINE_JOB_OUTPUT_LOCATION"]
instance_type = os.environ["INSTANCE_TYPE"]
instance_volume_size = int(os.environ["INSTANCE_VOLUME_SIZE"])
role_arn = os.environ["ROLE_ARN"]
kms_key_arn = os.environ.get("KMS_KEY_ARN")
stack_name = os.environ["STACK_NAME"]
max_runtime_seconds = int(os.environ["MAX_RUNTIME_SECONDS"])
try:
import os
import datetime
from shared.wrappers import BadRequest, api_exception_handler
from shared.logger import get_logger
from shared.helper import get_client
from lambda_helpers import (
validate,
template_url,
get_stack_name,
get_codepipeline_params,
get_image_builder_params,
format_template_parameters,
create_template_zip_file,
)
cloudformation_client = get_client("cloudformation")
codepipeline_client = get_client("codepipeline")
s3_client = get_client("s3")
logger = get_logger(__name__)
content_type = "plain/text"
# subclass JSONEncoder to be able to convert pipeline status to json
class DateTimeEncoder(JSONEncoder):
# Override the default method
def default(self, obj):
if isinstance(obj, (datetime.date, datetime.datetime)):
return obj.isoformat()
def test_reset_client(service):
get_client(service)
reset_client()
assert _helpers_service_clients == dict()
def test_get_client(service, enpoint_url):
client = get_client(service)
assert enpoint_url in client.meta.endpoint_url
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES #
# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions #
# and limitations under the License. #
# #####################################################################################################################
import os
import json
import boto3
from shared.wrappers import code_pipeline_exception_handler
from shared.logger import get_logger
from shared.helper import get_client
logger = get_logger(__name__)
lm_client = get_client("lambda")
cp_client = get_client("codepipeline")
@code_pipeline_exception_handler
def handler(event, context):
# todo: change the way to mock boto3 clients for unit tests without passing clients in input
# Extract the Job ID
job_id = event["CodePipeline.job"]["id"]
# Extract the Job Data
job_data = event["CodePipeline.job"]["data"]
user_parameters = job_data["actionConfiguration"]["configuration"][
"UserParameters"]
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES #
# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions #
# and limitations under the License. #
# #####################################################################################################################
import os
import json
import boto3
from shared.wrappers import api_exception_handler
from shared.logger import get_logger
from shared.helper import get_client
logger = get_logger(__name__)
sagemaker_client = get_client("sagemaker-runtime")
@api_exception_handler
def handler(event, context):
event_body = json.loads(event["body"])
endpoint_name = os.environ["SAGEMAKER_ENDPOINT_NAME"]
return invoke(event_body, endpoint_name)
def invoke(event_body, endpoint_name, sm_client=sagemaker_client):
response = sm_client.invoke_endpoint(
EndpointName=endpoint_name,
Body=event_body["payload"],
ContentType=event_body["content_type"])
logger.info(response)
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES #
# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions #
# and limitations under the License. #
# #####################################################################################################################
import logging
import uuid
import json
from crhelper import CfnResource
from shared.helper import get_client
logger = logging.getLogger(__name__)
lambda_client = get_client("lambda")
helper = CfnResource(json_logging=True, log_level="INFO")
def handler(event, context):
helper(event, context)
@helper.update
@helper.create
def invoke_lambda(event, _, lm_client=lambda_client):
try:
logger.info(f"Event received: {event}")
resource_properties = event["ResourceProperties"]
resource = resource_properties["Resource"]
if resource == "InvokeLambda":
# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES #
# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions #
# and limitations under the License. #
# #####################################################################################################################
import os
import json
import time
import botocore
import boto3
from shared.wrappers import code_pipeline_exception_handler
from shared.logger import get_logger
from shared.helper import get_client, get_built_in_model_monitor_container_uri
logger = get_logger(__name__)
sm_client = get_client("sagemaker")
cp_client = get_client("codepipeline")
@code_pipeline_exception_handler
def handler(event, context):
# Extract the Job ID
job_id = event["CodePipeline.job"]["id"]
baseline_job_name = os.environ["BASELINE_JOB_NAME"]
assets_bucket = os.environ["ASSETS_BUCKET"]
training_data_location = os.environ["TRAINING_DATA_LOCATION"]
baseline_job_output_location = os.environ["BASELINE_JOB_OUTPUT_LOCATION"]
instance_type = os.environ["INSTANCE_TYPE"]
instance_volume_size = int(os.environ["INSTANCE_VOLUME_SIZE"])
role_arn = os.environ["ROLE_ARN"]
find_artifact,
get_template,
put_job_failure,
start_stackset_update_or_create,
check_stackset_update_status,
get_user_params,
setup_s3_client,
)
from shared.logger import get_logger
from shared.helper import get_client
logger = get_logger(__name__)
logger.info("Loading stackset helpers...")
cf_client = get_client("cloudformation")
cp_client = get_client("codepipeline")
def lambda_handler(event, context):
"""The Lambda function handler
If a continuing job then checks the CloudFormation stackset and its instances status
and updates the job accordingly.
If a new job then kick of an update or creation of the target
CloudFormation stackset and its instances.
Args:
event: The event passed by Lambda
context: The context passed by Lambda
