def test_deploy_model_with_update_non_existing_endpoint( mxnet_training_job, sagemaker_session, mxnet_full_version, cpu_instance_type, alternative_cpu_instance_type, ): endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp()) expected_error_message = ( 'Endpoint with name "{}" does not exist; ' "please use an existing endpoint name".format(endpoint_name) ) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job ) model_data = desc["ModelArtifacts"]["S3ModelArtifacts"] script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py") model = MXNetModel( model_data, "SageMakerRole", entry_point=script_path, py_version=PYTHON_VERSION, sagemaker_session=sagemaker_session, framework_version=mxnet_full_version, ) model.deploy(1, alternative_cpu_instance_type, endpoint_name=endpoint_name) sagemaker_session.sagemaker_client.describe_endpoint(EndpointName=endpoint_name) with pytest.raises(ValueError, message=expected_error_message): model.deploy( 1, cpu_instance_type, update_endpoint=True, endpoint_name="non-existing-endpoint" )
def test_deploy_model_with_update_endpoint( mxnet_training_job, sagemaker_session, mxnet_full_version ): endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job ) model_data = desc["ModelArtifacts"]["S3ModelArtifacts"] script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py") model = MXNetModel( model_data, "SageMakerRole", entry_point=script_path, py_version=PYTHON_VERSION, sagemaker_session=sagemaker_session, framework_version=mxnet_full_version, ) model.deploy(1, "ml.t2.medium", endpoint_name=endpoint_name) old_endpoint = sagemaker_session.describe_endpoint(EndpointName=endpoint_name) old_config_name = old_endpoint["EndpointConfigName"] model.deploy(1, "ml.m4.xlarge", update_endpoint=True, endpoint_name=endpoint_name) new_endpoint = sagemaker_session.describe_endpoint(EndpointName=endpoint_name)[ "ProductionVariants" ] new_production_variants = new_endpoint["ProductionVariants"] new_config_name = new_endpoint["EndpointConfigName"] assert old_config_name != new_config_name assert new_production_variants["InstanceType"] == "ml.m4.xlarge" assert new_production_variants["InitialInstanceCount"] == 1 assert new_production_variants["AcceleratorType"] is None
def test_deploy_model_with_update_non_existing_endpoint( mxnet_training_job, sagemaker_session, mxnet_full_version): endpoint_name = 'test-mxnet-deploy-model-{}'.format(sagemaker_timestamp()) expected_error_message = 'Endpoint with name "{}" does not exist; ' \ 'please use an existing endpoint name'.format(endpoint_name) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job) model_data = desc['ModelArtifacts']['S3ModelArtifacts'] script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py') model = MXNetModel(model_data, 'SageMakerRole', entry_point=script_path, py_version=PYTHON_VERSION, sagemaker_session=sagemaker_session, framework_version=mxnet_full_version) model.deploy(1, 'ml.t2.medium', endpoint_name=endpoint_name) sagemaker_session.describe_endpoint(EndpointName=endpoint_name) with pytest.raises(ValueError, message=expected_error_message): model.deploy(1, 'ml.m4.xlarge', update_endpoint=True, endpoint_name='non-existing-endpoint')
def test_deploy_model_with_update_endpoint(mxnet_training_job, sagemaker_session, mxnet_full_version): endpoint_name = 'test-mxnet-deploy-model-{}'.format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job) model_data = desc['ModelArtifacts']['S3ModelArtifacts'] script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py') model = MXNetModel(model_data, 'SageMakerRole', entry_point=script_path, py_version=PYTHON_VERSION, sagemaker_session=sagemaker_session, framework_version=mxnet_full_version) model.deploy(1, 'ml.t2.medium', endpoint_name=endpoint_name) old_endpoint = sagemaker_session.describe_endpoint( EndpointName=endpoint_name) old_config_name = old_endpoint['EndpointConfigName'] model.deploy(1, 'ml.m4.xlarge', update_endpoint=True, endpoint_name=endpoint_name) new_endpoint = sagemaker_session.describe_endpoint( EndpointName=endpoint_name)['ProductionVariants'] new_production_variants = new_endpoint['ProductionVariants'] new_config_name = new_endpoint['EndpointConfigName'] assert old_config_name != new_config_name assert new_production_variants['InstanceType'] == 'ml.m4.xlarge' assert new_production_variants['InitialInstanceCount'] == 1 assert new_production_variants['AcceleratorType'] is None
def test_deploy_model_with_update_endpoint( mxnet_training_job, sagemaker_session, mxnet_full_version, cpu_instance_type, alternative_cpu_instance_type, ): endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job) model_data = desc["ModelArtifacts"]["S3ModelArtifacts"] script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py") model = MXNetModel( model_data, "SageMakerRole", entry_point=script_path, py_version=PYTHON_VERSION, sagemaker_session=sagemaker_session, framework_version=mxnet_full_version, ) model.deploy(1, alternative_cpu_instance_type, endpoint_name=endpoint_name) old_endpoint = sagemaker_session.sagemaker_client.describe_endpoint( EndpointName=endpoint_name) old_config_name = old_endpoint["EndpointConfigName"] model.deploy(1, cpu_instance_type, update_endpoint=True, endpoint_name=endpoint_name) # Wait for endpoint to finish updating max_retry_count = 40 # Endpoint update takes ~7min. 40 retries * 30s sleeps = 20min timeout current_retry_count = 0 while current_retry_count <= max_retry_count: if current_retry_count >= max_retry_count: raise Exception( "Endpoint status not 'InService' within expected timeout.") time.sleep(30) new_endpoint = sagemaker_session.sagemaker_client.describe_endpoint( EndpointName=endpoint_name) current_retry_count += 1 if new_endpoint["EndpointStatus"] == "InService": break new_config_name = new_endpoint["EndpointConfigName"] new_config = sagemaker_session.sagemaker_client.describe_endpoint_config( EndpointConfigName=new_config_name) assert old_config_name != new_config_name assert new_config["ProductionVariants"][0][ "InstanceType"] == cpu_instance_type assert new_config["ProductionVariants"][0]["InitialInstanceCount"] == 1
def test_deploy_model_with_tags_and_kms( mxnet_training_job, sagemaker_session, mxnet_inference_latest_version, mxnet_inference_latest_py_version, cpu_instance_type, ): endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job ) model_data = desc["ModelArtifacts"]["S3ModelArtifacts"] script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py") model = MXNetModel( model_data, "SageMakerRole", entry_point=script_path, py_version=mxnet_inference_latest_py_version, sagemaker_session=sagemaker_session, framework_version=mxnet_inference_latest_version, ) tags = [{"Key": "TagtestKey", "Value": "TagtestValue"}] kms_key_arn = get_or_create_kms_key(sagemaker_session) model.deploy( 1, cpu_instance_type, endpoint_name=endpoint_name, tags=tags, kms_key=kms_key_arn ) returned_model = sagemaker_session.sagemaker_client.describe_model(ModelName=model.name) returned_model_tags = sagemaker_session.sagemaker_client.list_tags( ResourceArn=returned_model["ModelArn"] )["Tags"] endpoint = sagemaker_session.sagemaker_client.describe_endpoint(EndpointName=endpoint_name) endpoint_tags = sagemaker_session.sagemaker_client.list_tags( ResourceArn=endpoint["EndpointArn"] )["Tags"] endpoint_config = sagemaker_session.sagemaker_client.describe_endpoint_config( EndpointConfigName=endpoint["EndpointConfigName"] ) endpoint_config_tags = sagemaker_session.sagemaker_client.list_tags( ResourceArn=endpoint_config["EndpointConfigArn"] )["Tags"] production_variants = endpoint_config["ProductionVariants"] assert returned_model_tags == tags assert endpoint_config_tags == tags assert endpoint_tags == tags assert production_variants[0]["InstanceType"] == cpu_instance_type assert production_variants[0]["InitialInstanceCount"] == 1 assert endpoint_config["KmsKeyId"] == kms_key_arn
def test_deploy_model_with_tags_and_kms(mxnet_training_job, sagemaker_session, mxnet_full_version): endpoint_name = 'test-mxnet-deploy-model-{}'.format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job) model_data = desc['ModelArtifacts']['S3ModelArtifacts'] script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py') model = MXNetModel(model_data, 'SageMakerRole', entry_point=script_path, py_version=PYTHON_VERSION, sagemaker_session=sagemaker_session, framework_version=mxnet_full_version) tags = [{'Key': 'TagtestKey', 'Value': 'TagtestValue'}] kms_key_arn = get_or_create_kms_key(sagemaker_session) model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name, tags=tags, kms_key=kms_key_arn) returned_model = sagemaker_session.describe_model( EndpointName=model.name) returned_model_tags = sagemaker_session.list_tags( ResourceArn=returned_model['ModelArn'])['Tags'] endpoint = sagemaker_session.describe_endpoint( EndpointName=endpoint_name) endpoint_tags = sagemaker_session.list_tags( ResourceArn=endpoint['EndpointArn'])['Tags'] endpoint_config = sagemaker_session.describe_endpoint_config( EndpointConfigName=endpoint['EndpointConfigName']) endpoint_config_tags = sagemaker_session.list_tags( ResourceArn=endpoint_config['EndpointConfigArn'])['Tags'] production_variants = endpoint_config['ProductionVariants'] assert returned_model_tags == tags assert endpoint_config_tags == tags assert endpoint_tags == tags assert production_variants[0]['InstanceType'] == 'ml.m4.xlarge' assert production_variants[0]['InitialInstanceCount'] == 1 assert endpoint_config['KmsKeyId'] == kms_key_arn
def test_gluonnlp(sagemaker_session, ecr_image, instance_type, framework_version): import urllib.request tmpdir = tempfile.mkdtemp() tmpfile = os.path.join(tmpdir, 'bert_sst.tar.gz') urllib.request.urlretrieve( 'https://aws-dlc-sample-models.s3.amazonaws.com/bert_sst/bert_sst.tar.gz', tmpfile) prefix = 'gluonnlp-serving/default-handlers' model_data = sagemaker_session.upload_data(path=tmpfile, key_prefix=prefix) model = MXNetModel(model_data, 'SageMakerRole', SCRIPT_PATH, image=ecr_image, py_version="py3", framework_version=framework_version, sagemaker_session=sagemaker_session) endpoint_name = utils.unique_name_from_base('test-mxnet-gluonnlp') with timeout.timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): predictor = model.deploy(1, instance_type, endpoint_name=endpoint_name) output = predictor.predict( ["Positive sentiment", "Negative sentiment"]) assert [1, 0] == output
def deploy_model(model_data, endpoint_name, sagemaker_session, ecr_image, instance_type, framework_version, role, script="bert_inference.py", source_dir="code/"): """ """ if ecr_image is not None: model = MXNetModel(model_data, role, entry_point=script, source_dir=source_dir, image_uri=ecr_image, py_version="py3", framework_version=framework_version, sagemaker_session=sagemaker_session) else: model = MXNetModel(model_data, role, entry_point=script, source_dir=source_dir, py_version="py3", framework_version=framework_version, sagemaker_session=sagemaker_session) endpoint_name = utils.unique_name_from_base(endpoint_name) print("Deploying...") predictor = model.deploy(1, instance_type, endpoint_name=endpoint_name) print("\nEndpoint Name: {}".format(endpoint_name)) return predictor
def test_neuron_hosting(sagemaker_session, ecr_image, instance_type, framework_version, skip_if_no_neuron): prefix = "mxnet-serving/neuron-handlers" model_data = sagemaker_session.upload_data(path=MODEL_PATH, key_prefix=prefix) # Since fake inf is run in mnist-neuron.py and NEURONCORE_GROUP_SIZE env variable # is not set, set the model server worker to 1 (or need to set env variable). # Otherwise num workers defaults to 4 (4 cpu's) and run out of neuron cores model = MXNetModel( model_data, "SageMakerRole", SCRIPT_PATH, framework_version=framework_version, image_uri=ecr_image, model_server_workers=1, sagemaker_session=sagemaker_session, ) endpoint_name = utils.unique_name_from_base("test-neuron-mxnet-serving") with timeout.timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): predictor = model.deploy(1, instance_type, endpoint_name=endpoint_name) numpy_ndarray = np.load(INPUT_PATH) output = predictor.predict(data=numpy_ndarray) with open(OUTPUT_PATH) as outputfile: expected_output = json.load(outputfile) assert expected_output == output print(output)
def test_neuron_hosting(sagemaker_session, ecr_image, instance_type, framework_version, skip_if_no_neuron): prefix = "mxnet-serving/neuron-handlers" model_data = sagemaker_session.upload_data(path=MODEL_PATH, key_prefix=prefix) model = MXNetModel( model_data, "SageMakerRole", SCRIPT_PATH, framework_version=framework_version, image_uri=ecr_image, sagemaker_session=sagemaker_session, ) endpoint_name = utils.unique_name_from_base("test-neuron-mxnet-serving") with timeout.timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): predictor = model.deploy(1, instance_type, endpoint_name=endpoint_name) numpy_ndarray = np.load(INPUT_PATH) output = predictor.predict(data=numpy_ndarray) with open(OUTPUT_PATH) as outputfile: expected_output = json.load(outputfile) assert expected_output == output print(output)
def test_gluoncv(sagemaker_session, ecr_image, instance_type, framework_version): try: # python3 from urllib.request import urlretrieve except: # python2 from urllib import urlretrieve tmpdir = tempfile.mkdtemp() tmpfile = os.path.join(tmpdir, 'yolo3_darknet53_voc.tar.gz') urlretrieve('https://dlc-samples.s3.amazonaws.com/mxnet/gluon/yolo3_darknet53_voc.tar.gz', tmpfile) prefix = 'gluoncv-serving/default-handlers' model_data = sagemaker_session.upload_data(path=tmpfile, key_prefix=prefix) model = MXNetModel(model_data, 'SageMakerRole', SCRIPT_PATH, image=ecr_image, py_version="py3", framework_version=framework_version, sagemaker_session=sagemaker_session) endpoint_name = utils.unique_name_from_base('test-mxnet-gluoncv') with timeout.timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): predictor = model.deploy(1, instance_type, endpoint_name=endpoint_name) with open(SCRIPT_DATA_PATH, 'rb') as fdata: output = predictor.predict([fdata.read().hex()]) assert len(output[0]) == 100
def test_deploy_model(mxnet_training_job, sagemaker_session): endpoint_name = 'test-mxnet-deploy-model-{}'.format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job) model_data = desc['ModelArtifacts']['S3ModelArtifacts'] script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist_neo.py') role = 'SageMakerRole' model = MXNetModel(model_data, role, entry_point=script_path, py_version=PYTHON_VERSION, sagemaker_session=sagemaker_session) model.compile(target_instance_family='ml_m4', input_shape={'data': [1, 1, 28, 28]}, role=role, job_name='test-deploy-model-compilation-job-{}'.format( int(time.time())), output_path='/'.join(model_data.split('/')[:-1])) predictor = model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name) predictor.content_type = "application/vnd+python.numpy+binary" data = numpy.zeros(shape=(1, 1, 28, 28)) predictor.predict(data)
def test_deploy_model_with_accelerator( mxnet_training_job, sagemaker_session, ei_mxnet_full_version ): endpoint_name = "test-mxnet-deploy-model-ei-{}".format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job ) model_data = desc["ModelArtifacts"]["S3ModelArtifacts"] script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py") model = MXNetModel( model_data, "SageMakerRole", entry_point=script_path, framework_version=ei_mxnet_full_version, py_version=PYTHON_VERSION, sagemaker_session=sagemaker_session, ) predictor = model.deploy( 1, "ml.m4.xlarge", endpoint_name=endpoint_name, accelerator_type="ml.eia1.medium" ) data = numpy.zeros(shape=(1, 1, 28, 28)) result = predictor.predict(data) assert result is not None
def test_deploy_model(mxnet_training_job, sagemaker_session, mxnet_full_version): endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job ) model_data = desc["ModelArtifacts"]["S3ModelArtifacts"] script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py") model = MXNetModel( model_data, "SageMakerRole", entry_point=script_path, py_version=PYTHON_VERSION, sagemaker_session=sagemaker_session, framework_version=mxnet_full_version, ) predictor = model.deploy(1, "ml.m4.xlarge", endpoint_name=endpoint_name) data = numpy.zeros(shape=(1, 1, 28, 28)) result = predictor.predict(data) assert result is not None predictor.delete_model() with pytest.raises(Exception) as exception: sagemaker_session.sagemaker_client.describe_model(ModelName=model.name) assert "Could not find model" in str(exception.value)
def test_deploy_model(mxnet_training_job, sagemaker_session, cpu_instance_type, cpu_instance_family): endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job) model_data = desc["ModelArtifacts"]["S3ModelArtifacts"] script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist_neo.py") role = "SageMakerRole" model = MXNetModel( model_data, role, entry_point=script_path, py_version=PYTHON_VERSION, sagemaker_session=sagemaker_session, ) model.compile( target_instance_family=cpu_instance_family, input_shape={"data": [1, 1, 28, 28]}, role=role, job_name="test-deploy-model-compilation-job-{}".format( int(time.time())), output_path="/".join(model_data.split("/")[:-1]), ) predictor = model.deploy(1, cpu_instance_type, endpoint_name=endpoint_name) predictor.content_type = "application/vnd+python.numpy+binary" data = numpy.zeros(shape=(1, 1, 28, 28)) predictor.predict(data)
def test_deploy_model(mxnet_training_job, sagemaker_session, mxnet_full_version): endpoint_name = 'test-mxnet-deploy-model-{}'.format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job) model_data = desc['ModelArtifacts']['S3ModelArtifacts'] script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py') model = MXNetModel(model_data, 'SageMakerRole', entry_point=script_path, py_version=PYTHON_VERSION, sagemaker_session=sagemaker_session, framework_version=mxnet_full_version) predictor = model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name) data = numpy.zeros(shape=(1, 1, 28, 28)) predictor.predict(data) predictor.delete_model() with pytest.raises(Exception) as exception: sagemaker_session.sagemaker_client.describe_model(ModelName=model.name) assert 'Could not find model' in str(exception.value)
def test_inferentia_deploy_model(mxnet_training_job, sagemaker_session, inf_instance_type, inf_instance_family): endpoint_name = unique_name_from_base("test-neo-deploy-model") with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job) model_data = desc["ModelArtifacts"]["S3ModelArtifacts"] script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist_neo.py") role = "SageMakerRole" model = MXNetModel( model_data, role, entry_point=script_path, framework_version=INF_MXNET_VERSION, sagemaker_session=sagemaker_session, ) model.compile( target_instance_family=inf_instance_family, input_shape={"data": [1, 1, 28, 28]}, role=role, job_name=unique_name_from_base( "test-deploy-model-compilation-job"), output_path="/".join(model_data.split("/")[:-1]), ) predictor = model.deploy(1, inf_instance_type, endpoint_name=endpoint_name) predictor.content_type = "application/vnd+python.numpy+binary" data = numpy.zeros(shape=(1, 1, 28, 28)) predictor.predict(data)
def test_git_support_codecommit_with_mxnet(sagemaker_local_session): script_path = "mnist.py" data_path = os.path.join(DATA_DIR, "mxnet_mnist") git_config = { "repo": CODECOMMIT_REPO, "branch": CODECOMMIT_BRANCH, "username": "******", "password": "******", } source_dir = "mxnet" dependencies = ["foo/bar.py"] mx = MXNet( entry_point=script_path, role="SageMakerRole", source_dir=source_dir, dependencies=dependencies, framework_version=MXNet.LATEST_VERSION, py_version=PYTHON_VERSION, train_instance_count=1, train_instance_type="local", sagemaker_session=sagemaker_local_session, git_config=git_config, ) mx.fit( { "train": "file://" + os.path.join(data_path, "train"), "test": "file://" + os.path.join(data_path, "test"), } ) files = [file for file in os.listdir(mx.source_dir)] assert "some_file" in files assert "mnist.py" in files assert os.path.exists(mx.dependencies[0]) with lock.lock(LOCK_PATH): try: client = sagemaker_local_session.sagemaker_client desc = client.describe_training_job(TrainingJobName=mx.latest_training_job.name) model_data = desc["ModelArtifacts"]["S3ModelArtifacts"] model = MXNetModel( model_data, "SageMakerRole", entry_point=script_path, source_dir=source_dir, dependencies=dependencies, py_version=PYTHON_VERSION, sagemaker_session=sagemaker_local_session, framework_version=MXNet.LATEST_VERSION, git_config=git_config, ) predictor = model.deploy(1, "local") data = numpy.zeros(shape=(1, 1, 28, 28)) result = predictor.predict(data) assert result is not None finally: predictor.delete_endpoint()
def test_deploy_model(mxnet_training_job, sagemaker_session): endpoint_name = 'test-mxnet-deploy-model-{}'.format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job(TrainingJobName=mxnet_training_job) model_data = desc['ModelArtifacts']['S3ModelArtifacts'] script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py') model = MXNetModel(model_data, 'SageMakerRole', entry_point=script_path, sagemaker_session=sagemaker_session) predictor = model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name) data = numpy.zeros(shape=(1, 1, 28, 28)) predictor.predict(data)
def predictor(docker_image, sagemaker_local_session, local_instance_type): default_handler_path = os.path.join(RESOURCE_PATH, 'default_handlers') m = MXNetModel('file://{}'.format(os.path.join(default_handler_path, 'model')), 'SageMakerRole', os.path.join(default_handler_path, 'code', 'empty_module.py'), image=docker_image, sagemaker_session=sagemaker_local_session, model_server_workers=NUM_MODEL_SERVER_WORKERS) with local_mode_utils.lock(): try: predictor = m.deploy(1, local_instance_type) yield predictor finally: sagemaker_local_session.delete_endpoint(m.endpoint_name)
def test_deploy_model(mxnet_training_job, sagemaker_session): endpoint_name = 'test-mxnet-deploy-model-{}'.format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, minutes=20): desc = sagemaker_session.sagemaker_client.describe_training_job(TrainingJobName=mxnet_training_job) model_data = desc['ModelArtifacts']['S3ModelArtifacts'] script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py') model = MXNetModel(model_data, 'SageMakerRole', entry_point=script_path, sagemaker_session=sagemaker_session) predictor = model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name) data = numpy.zeros(shape=(1, 1, 28, 28)) predictor.predict(data)
def predictor(docker_image, sagemaker_local_session, local_instance_type): model = MXNetModel('file://{}'.format(MODEL_PATH), 'SageMakerRole', SCRIPT_PATH, image=docker_image, sagemaker_session=sagemaker_local_session) with local_mode_utils.lock(): try: predictor = model.deploy(1, local_instance_type) yield predictor finally: predictor.delete_endpoint()
def test_deploy_model_with_accelerator(mxnet_training_job, sagemaker_session, ei_mxnet_full_version): endpoint_name = 'test-mxnet-deploy-model-ei-{}'.format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job(TrainingJobName=mxnet_training_job) model_data = desc['ModelArtifacts']['S3ModelArtifacts'] script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py') model = MXNetModel(model_data, 'SageMakerRole', entry_point=script_path, framework_version=ei_mxnet_full_version, py_version=PYTHON_VERSION, sagemaker_session=sagemaker_session) predictor = model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name, accelerator_type='ml.eia1.medium') data = numpy.zeros(shape=(1, 1, 28, 28)) predictor.predict(data)
def test_hosting(docker_image, sagemaker_local_session, local_instance_type): model = MXNetModel('file://{}'.format(MODEL_PATH), 'SageMakerRole', SCRIPT_PATH, image=docker_image, sagemaker_session=sagemaker_local_session) input = json.dumps({'some': 'json'}) with local_mode_utils.lock(): try: predictor = model.deploy(1, local_instance_type) output = predictor.predict(input) assert input == output finally: sagemaker_local_session.delete_endpoint(model.endpoint_name)
def test_hosting(docker_image, sagemaker_local_session, local_instance_type): hosting_resource_path = os.path.join(RESOURCE_PATH, 'dummy_hosting') m = MXNetModel('file://{}'.format(os.path.join(hosting_resource_path, 'code')), 'SageMakerRole', os.path.join(hosting_resource_path, 'code', 'dummy_hosting_module.py'), image=docker_image, sagemaker_session=sagemaker_local_session, model_server_workers=NUM_MODEL_SERVER_WORKERS) input = json.dumps({'some': 'json'}) with local_mode_utils.lock(): try: predictor = m.deploy(1, local_instance_type) output = predictor.predict(input) assert input == output finally: sagemaker_local_session.delete_endpoint(m.endpoint_name)
def _test_hosting_function(ecr_image, sagemaker_session, instance_type, framework_version): prefix = 'mxnet-serving/default-handlers' model_data = sagemaker_session.upload_data(path=MODEL_PATH, key_prefix=prefix) model = MXNetModel(model_data, 'SageMakerRole', SCRIPT_PATH, framework_version=framework_version, image_uri=ecr_image, sagemaker_session=sagemaker_session) endpoint_name = utils.unique_name_from_base('test-mxnet-serving') with timeout.timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): predictor = model.deploy(1, instance_type, endpoint_name=endpoint_name) output = predictor.predict([[1, 2]]) assert [[4.9999918937683105]] == output
def test_deploy_model( mxnet_training_job, sagemaker_session, cpu_instance_type, cpu_instance_family, neo_mxnet_latest_version, neo_mxnet_latest_py_version, ): endpoint_name = unique_name_from_base("test-neo-deploy-model") with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job) model_data = desc["ModelArtifacts"]["S3ModelArtifacts"] script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist_neo.py") role = "SageMakerRole" model = MXNetModel( model_data, role, entry_point=script_path, py_version=neo_mxnet_latest_py_version, framework_version=neo_mxnet_latest_version, sagemaker_session=sagemaker_session, ) serializer = JSONSerializer() serializer.CONTENT_TYPE = "application/vnd+python.numpy+binary" model.compile( target_instance_family=cpu_instance_family, input_shape={ "data": [1, 1, 28, 28], "softmax_label": [1] }, role=role, job_name=unique_name_from_base( "test-deploy-model-compilation-job"), output_path="/".join(model_data.split("/")[:-1]), ) predictor = model.deploy(1, cpu_instance_type, serializer=serializer, endpoint_name=endpoint_name) data = numpy.zeros(shape=(1, 1, 28, 28)) predictor.predict(data)
def test_gluon_hosting(docker_image, sagemaker_local_session, local_instance_type): model = MXNetModel('file://{}'.format(MODEL_PATH), 'SageMakerRole', SCRIPT_PATH, image=docker_image, sagemaker_session=sagemaker_local_session) with open(os.path.join(RESOURCE_PATH, 'mnist', 'images', '04.json'), 'r') as f: input = json.load(f) with local_mode_utils.lock(): try: predictor = model.deploy(1, local_instance_type) output = predictor.predict(input) assert [4.0] == output finally: predictor.delete_endpoint()
def test_register_model_package_versioned( mxnet_training_job, sagemaker_session, mxnet_inference_latest_version, mxnet_inference_latest_py_version, cpu_instance_type, ): endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job) model_package_group_name = "register-model-package-{}".format( sagemaker_timestamp()) sagemaker_session.sagemaker_client.create_model_package_group( ModelPackageGroupName=model_package_group_name) model_data = desc["ModelArtifacts"]["S3ModelArtifacts"] script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py") model = MXNetModel( model_data, "SageMakerRole", entry_point=script_path, py_version=mxnet_inference_latest_py_version, sagemaker_session=sagemaker_session, framework_version=mxnet_inference_latest_version, ) model_pkg = model.register( content_types=["application/json"], response_types=["application/json"], inference_instances=["ml.m5.large"], transform_instances=["ml.m5.large"], model_package_group_name=model_package_group_name, approval_status="Approved", ) assert isinstance(model_pkg, ModelPackage) predictor = model.deploy(1, cpu_instance_type, endpoint_name=endpoint_name) data = numpy.zeros(shape=(1, 1, 28, 28)) result = predictor.predict(data) assert result is not None sagemaker_session.sagemaker_client.delete_model_package( ModelPackageName=model_pkg.model_package_arn) sagemaker_session.sagemaker_client.delete_model_package_group( ModelPackageGroupName=model_package_group_name)
def test_deploy_model_and_update_endpoint( mxnet_training_job, sagemaker_session, mxnet_inference_latest_version, mxnet_inference_latest_py_version, cpu_instance_type, alternative_cpu_instance_type, ): endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp()) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session): desc = sagemaker_session.sagemaker_client.describe_training_job( TrainingJobName=mxnet_training_job) model_data = desc["ModelArtifacts"]["S3ModelArtifacts"] script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py") model = MXNetModel( model_data, "SageMakerRole", entry_point=script_path, py_version=mxnet_inference_latest_py_version, sagemaker_session=sagemaker_session, framework_version=mxnet_inference_latest_version, ) predictor = model.deploy(1, alternative_cpu_instance_type, endpoint_name=endpoint_name) endpoint_desc = sagemaker_session.sagemaker_client.describe_endpoint( EndpointName=endpoint_name) old_config_name = endpoint_desc["EndpointConfigName"] predictor.update_endpoint(initial_instance_count=1, instance_type=cpu_instance_type) endpoint_desc = sagemaker_session.sagemaker_client.describe_endpoint( EndpointName=endpoint_name) new_config_name = endpoint_desc["EndpointConfigName"] new_config = sagemaker_session.sagemaker_client.describe_endpoint_config( EndpointConfigName=new_config_name) assert old_config_name != new_config_name assert new_config["ProductionVariants"][0][ "InstanceType"] == cpu_instance_type assert new_config["ProductionVariants"][0]["InitialInstanceCount"] == 1