def test_handler_success( sm_describe_endpoint_config_params, sm_create_endpoint_config_params, sm_create_endpoint_config_response, cp_expected_params, sm_describe_endpoint_params, sm_create_endpoint_params, sm_create_endpoint_response, sm_describe_endpoint_response_2, event): sm_client = get_client("sagemaker") cp_client = get_client("codepipeline") sm_stubber = Stubber(sm_client) cp_stubber = Stubber(cp_client) # endpoint config creation sm_describe_endpoint_config_response = {} cp_response = {} sm_stubber.add_client_error( "describe_endpoint_config", service_error_code="EndpointConfigExists", service_message="Could not find endpoint configuration", http_status_code=400, expected_params=sm_describe_endpoint_config_params, ) sm_stubber.add_response( "create_endpoint_config", sm_create_endpoint_config_response, sm_create_endpoint_config_params, ) # endpoint creation sm_stubber.add_client_error( "describe_endpoint", service_error_code="EndpointExists", service_message="Could not find endpoint", http_status_code=400, expected_params=sm_describe_endpoint_params, ) sm_stubber.add_response("create_endpoint", sm_create_endpoint_response, sm_create_endpoint_params) sm_stubber.add_response( "describe_endpoint", sm_describe_endpoint_response_2, sm_describe_endpoint_params, ) cp_stubber.add_response("put_job_success_result", cp_response, cp_expected_params) expected_log_message = ( "Sent success message back to codepipeline with job_id: test_job_id") with sm_stubber: with cp_stubber: handler(event, {}) cp_stubber.assert_no_pending_responses() reset_client()
def test_handler_success( sm_create_monitoring_expected_params, sm_create_monitoring_response_200, sm_describe_monitoring_schedule_response, cp_expected_params_success, sm_describe_monitoring_scheduale_params, event, ): sm_client = get_client("sagemaker") cp_client = get_client("codepipeline") sm_stubber = Stubber(sm_client) cp_stubber = Stubber(cp_client) cp_response = {} # job creation sm_stubber.add_client_error( "describe_monitoring_schedule", service_error_code="MonitorJobExists", service_message="Could not find requested job with name", http_status_code=400, expected_params=sm_describe_monitoring_scheduale_params, ) # success path sm_stubber.add_response("create_monitoring_schedule", sm_create_monitoring_response_200, sm_create_monitoring_expected_params) sm_stubber.add_response( "describe_monitoring_schedule", sm_describe_monitoring_schedule_response, sm_describe_monitoring_scheduale_params, ) cp_stubber.add_response("put_job_success_result", cp_response, cp_expected_params_success) with sm_stubber: with cp_stubber: handler(event, {}) cp_stubber.assert_no_pending_responses() reset_client()
def test_handler_failure(lm_expected_params, lm_response_500, cp_expected_params_failure, event): lm_client = get_client("lambda") lm_stubber = Stubber(lm_client) cp_client = get_client("codepipeline") cp_stubber = Stubber(cp_client) cp_response = {} lm_stubber.add_response("update_function_configuration", lm_response_500, lm_expected_params) cp_stubber.add_response("put_job_failure_result", cp_response, cp_expected_params_failure) with lm_stubber: with cp_stubber: handler(event, {}) cp_stubber.assert_no_pending_responses() reset_client()
def test_handler_fail(sm_expected_params, sm_response_500, cp_expected_params_failure, event): sm_client = get_client("sagemaker") cp_client = get_client("codepipeline") sm_stubber = Stubber(sm_client) cp_stubber = Stubber(cp_client) cp_response = {} # fail path sm_stubber.add_response("create_transform_job", sm_response_500, sm_expected_params) cp_stubber.add_response("put_job_failure_result", cp_response, cp_expected_params_failure) with sm_stubber: with cp_stubber: handler(event, {}) cp_stubber.assert_no_pending_responses() reset_client()
def test_handler_success(sm_expected_params, sm_response_200, event): sm_client = get_client("sagemaker") sm_stubber = Stubber(sm_client) # success path sm_stubber.add_response("create_transform_job", sm_response_200, sm_expected_params) with sm_stubber: handler(event, {}) reset_client()
def test_handler_fail(sm_expected_params, sm_response_500, event): sm_client = get_client("sagemaker") sm_stubber = Stubber(sm_client) # fail path sm_stubber.add_response("create_transform_job", sm_response_500, sm_expected_params) with pytest.raises(Exception): handler(event, {}) reset_client()
def test_handler_success( sm_describe_model_expected_params, sm_describe_model_response, sm_delete_model_expected_params, sm_create_model_expected_params, sm_create_model_response, cp_expected_params, event, ): sm_client = get_client("sagemaker") cp_client = get_client("codepipeline") sm_stubber = Stubber(sm_client) cp_stubber = Stubber(cp_client) # describe model sm_stubber.add_response("describe_model", sm_describe_model_response, sm_describe_model_expected_params) # delete model sm_delete_model_response = {} sm_stubber.add_response("delete_model", sm_delete_model_response, sm_delete_model_expected_params) # create model sm_stubber.add_response("create_model", sm_create_model_response, sm_create_model_expected_params) # codepipeline cp_response = {} cp_stubber.add_response("put_job_success_result", cp_response, cp_expected_params) with sm_stubber: with cp_stubber: handler(event, {}) cp_stubber.assert_no_pending_responses() reset_client()
def test_handler_success( sm_create_baseline_expected_params, sm_create_job_response_200, event, ): sm_client = get_client("sagemaker") sm_stubber = Stubber(sm_client) # success path sm_stubber.add_response("create_processing_job", sm_create_job_response_200, sm_create_baseline_expected_params) with sm_stubber: handler(event, {}) reset_client()
def wrapper(event, context): try: return f(event, context) except Exception as e: codepipeline = get_client("codepipeline") exc_type, exc_value, exc_tb = sys.exc_info() logger.error( traceback.format_exception(exc_type, exc_value, exc_tb)) codepipeline.put_job_failure_result( jobId=event["CodePipeline.job"]["id"], failureDetails={ "message": f"Job failed. {str(e)}. Check the logs for more info.", "type": "JobFailed", }, )
# with the License. A copy of the License is located at # # # # http://www.apache.org/licenses/LICENSE-2.0 # # # # or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # # and limitations under the License. # # ##################################################################################################################### import os import botocore import boto3 from shared.logger import get_logger from shared.helper import get_client, get_built_in_model_monitor_container_uri logger = get_logger(__name__) sm_client = get_client("sagemaker") def handler(event, context): baseline_job_name = os.environ["BASELINE_JOB_NAME"] assets_bucket = os.environ["ASSETS_BUCKET"] training_data_location = os.environ["TRAINING_DATA_LOCATION"] baseline_job_output_location = os.environ["BASELINE_JOB_OUTPUT_LOCATION"] instance_type = os.environ["INSTANCE_TYPE"] instance_volume_size = int(os.environ["INSTANCE_VOLUME_SIZE"]) role_arn = os.environ["ROLE_ARN"] kms_key_arn = os.environ.get("KMS_KEY_ARN") stack_name = os.environ["STACK_NAME"] max_runtime_seconds = int(os.environ["MAX_RUNTIME_SECONDS"]) try:
import os import datetime from shared.wrappers import BadRequest, api_exception_handler from shared.logger import get_logger from shared.helper import get_client from lambda_helpers import ( validate, template_url, get_stack_name, get_codepipeline_params, get_image_builder_params, format_template_parameters, create_template_zip_file, ) cloudformation_client = get_client("cloudformation") codepipeline_client = get_client("codepipeline") s3_client = get_client("s3") logger = get_logger(__name__) content_type = "plain/text" # subclass JSONEncoder to be able to convert pipeline status to json class DateTimeEncoder(JSONEncoder): # Override the default method def default(self, obj): if isinstance(obj, (datetime.date, datetime.datetime)): return obj.isoformat()
def test_reset_client(service): get_client(service) reset_client() assert _helpers_service_clients == dict()
def test_get_client(service, enpoint_url): client = get_client(service) assert enpoint_url in client.meta.endpoint_url
# http://www.apache.org/licenses/LICENSE-2.0 # # # # or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # # and limitations under the License. # # ##################################################################################################################### import os import json import boto3 from shared.wrappers import code_pipeline_exception_handler from shared.logger import get_logger from shared.helper import get_client logger = get_logger(__name__) lm_client = get_client("lambda") cp_client = get_client("codepipeline") @code_pipeline_exception_handler def handler(event, context): # todo: change the way to mock boto3 clients for unit tests without passing clients in input # Extract the Job ID job_id = event["CodePipeline.job"]["id"] # Extract the Job Data job_data = event["CodePipeline.job"]["data"] user_parameters = job_data["actionConfiguration"]["configuration"][ "UserParameters"]
# # # http://www.apache.org/licenses/LICENSE-2.0 # # # # or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # # and limitations under the License. # # ##################################################################################################################### import os import json import boto3 from shared.wrappers import api_exception_handler from shared.logger import get_logger from shared.helper import get_client logger = get_logger(__name__) sagemaker_client = get_client("sagemaker-runtime") @api_exception_handler def handler(event, context): event_body = json.loads(event["body"]) endpoint_name = os.environ["SAGEMAKER_ENDPOINT_NAME"] return invoke(event_body, endpoint_name) def invoke(event_body, endpoint_name, sm_client=sagemaker_client): response = sm_client.invoke_endpoint( EndpointName=endpoint_name, Body=event_body["payload"], ContentType=event_body["content_type"]) logger.info(response)
# # # http://www.apache.org/licenses/LICENSE-2.0 # # # # or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # # and limitations under the License. # # ##################################################################################################################### import logging import uuid import json from crhelper import CfnResource from shared.helper import get_client logger = logging.getLogger(__name__) lambda_client = get_client("lambda") helper = CfnResource(json_logging=True, log_level="INFO") def handler(event, context): helper(event, context) @helper.update @helper.create def invoke_lambda(event, _, lm_client=lambda_client): try: logger.info(f"Event received: {event}") resource_properties = event["ResourceProperties"] resource = resource_properties["Resource"] if resource == "InvokeLambda":
# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # # and limitations under the License. # # ##################################################################################################################### import os import json import time import botocore import boto3 from shared.wrappers import code_pipeline_exception_handler from shared.logger import get_logger from shared.helper import get_client, get_built_in_model_monitor_container_uri logger = get_logger(__name__) sm_client = get_client("sagemaker") cp_client = get_client("codepipeline") @code_pipeline_exception_handler def handler(event, context): # Extract the Job ID job_id = event["CodePipeline.job"]["id"] baseline_job_name = os.environ["BASELINE_JOB_NAME"] assets_bucket = os.environ["ASSETS_BUCKET"] training_data_location = os.environ["TRAINING_DATA_LOCATION"] baseline_job_output_location = os.environ["BASELINE_JOB_OUTPUT_LOCATION"] instance_type = os.environ["INSTANCE_TYPE"] instance_volume_size = int(os.environ["INSTANCE_VOLUME_SIZE"]) role_arn = os.environ["ROLE_ARN"]
find_artifact, get_template, put_job_failure, start_stackset_update_or_create, check_stackset_update_status, get_user_params, setup_s3_client, ) from shared.logger import get_logger from shared.helper import get_client logger = get_logger(__name__) logger.info("Loading stackset helpers...") cf_client = get_client("cloudformation") cp_client = get_client("codepipeline") def lambda_handler(event, context): """The Lambda function handler If a continuing job then checks the CloudFormation stackset and its instances status and updates the job accordingly. If a new job then kick of an update or creation of the target CloudFormation stackset and its instances. Args: event: The event passed by Lambda context: The context passed by Lambda