def test_deploy_model_with_update_non_existing_endpoint(
mxnet_training_job,
sagemaker_session,
mxnet_full_version,
cpu_instance_type,
alternative_cpu_instance_type,
):
endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp())
expected_error_message = (
'Endpoint with name "{}" does not exist; '
"please use an existing endpoint name".format(endpoint_name)
)
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job
)
model_data = desc["ModelArtifacts"]["S3ModelArtifacts"]
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py")
model = MXNetModel(
model_data,
"SageMakerRole",
entry_point=script_path,
py_version=PYTHON_VERSION,
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version,
)
model.deploy(1, alternative_cpu_instance_type, endpoint_name=endpoint_name)
sagemaker_session.sagemaker_client.describe_endpoint(EndpointName=endpoint_name)
with pytest.raises(ValueError, message=expected_error_message):
model.deploy(
1, cpu_instance_type, update_endpoint=True, endpoint_name="non-existing-endpoint"
)
def test_deploy_model_with_update_endpoint(
mxnet_training_job, sagemaker_session, mxnet_full_version
):
endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job
)
model_data = desc["ModelArtifacts"]["S3ModelArtifacts"]
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py")
model = MXNetModel(
model_data,
"SageMakerRole",
entry_point=script_path,
py_version=PYTHON_VERSION,
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version,
)
model.deploy(1, "ml.t2.medium", endpoint_name=endpoint_name)
old_endpoint = sagemaker_session.describe_endpoint(EndpointName=endpoint_name)
old_config_name = old_endpoint["EndpointConfigName"]
model.deploy(1, "ml.m4.xlarge", update_endpoint=True, endpoint_name=endpoint_name)
new_endpoint = sagemaker_session.describe_endpoint(EndpointName=endpoint_name)[
"ProductionVariants"
]
new_production_variants = new_endpoint["ProductionVariants"]
new_config_name = new_endpoint["EndpointConfigName"]
assert old_config_name != new_config_name
assert new_production_variants["InstanceType"] == "ml.m4.xlarge"
assert new_production_variants["InitialInstanceCount"] == 1
assert new_production_variants["AcceleratorType"] is None
def test_deploy_model_with_update_non_existing_endpoint(
mxnet_training_job, sagemaker_session, mxnet_full_version):
endpoint_name = 'test-mxnet-deploy-model-{}'.format(sagemaker_timestamp())
expected_error_message = 'Endpoint with name "{}" does not exist; ' \
'please use an existing endpoint name'.format(endpoint_name)
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job)
model_data = desc['ModelArtifacts']['S3ModelArtifacts']
script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py')
model = MXNetModel(model_data,
'SageMakerRole',
entry_point=script_path,
py_version=PYTHON_VERSION,
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version)
model.deploy(1, 'ml.t2.medium', endpoint_name=endpoint_name)
sagemaker_session.describe_endpoint(EndpointName=endpoint_name)
with pytest.raises(ValueError, message=expected_error_message):
model.deploy(1,
'ml.m4.xlarge',
update_endpoint=True,
endpoint_name='non-existing-endpoint')
def test_deploy_model_with_update_endpoint(mxnet_training_job,
sagemaker_session,
mxnet_full_version):
endpoint_name = 'test-mxnet-deploy-model-{}'.format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job)
model_data = desc['ModelArtifacts']['S3ModelArtifacts']
script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py')
model = MXNetModel(model_data,
'SageMakerRole',
entry_point=script_path,
py_version=PYTHON_VERSION,
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version)
model.deploy(1, 'ml.t2.medium', endpoint_name=endpoint_name)
old_endpoint = sagemaker_session.describe_endpoint(
EndpointName=endpoint_name)
old_config_name = old_endpoint['EndpointConfigName']
model.deploy(1,
'ml.m4.xlarge',
update_endpoint=True,
endpoint_name=endpoint_name)
new_endpoint = sagemaker_session.describe_endpoint(
EndpointName=endpoint_name)['ProductionVariants']
new_production_variants = new_endpoint['ProductionVariants']
new_config_name = new_endpoint['EndpointConfigName']
assert old_config_name != new_config_name
assert new_production_variants['InstanceType'] == 'ml.m4.xlarge'
assert new_production_variants['InitialInstanceCount'] == 1
assert new_production_variants['AcceleratorType'] is None
def test_deploy_model_with_update_endpoint(
mxnet_training_job,
sagemaker_session,
mxnet_full_version,
cpu_instance_type,
alternative_cpu_instance_type,
):
endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job)
model_data = desc["ModelArtifacts"]["S3ModelArtifacts"]
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py")
model = MXNetModel(
model_data,
"SageMakerRole",
entry_point=script_path,
py_version=PYTHON_VERSION,
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version,
)
model.deploy(1,
alternative_cpu_instance_type,
endpoint_name=endpoint_name)
old_endpoint = sagemaker_session.sagemaker_client.describe_endpoint(
EndpointName=endpoint_name)
old_config_name = old_endpoint["EndpointConfigName"]
model.deploy(1,
cpu_instance_type,
update_endpoint=True,
endpoint_name=endpoint_name)
# Wait for endpoint to finish updating
max_retry_count = 40 # Endpoint update takes ~7min. 40 retries * 30s sleeps = 20min timeout
current_retry_count = 0
while current_retry_count <= max_retry_count:
if current_retry_count >= max_retry_count:
raise Exception(
"Endpoint status not 'InService' within expected timeout.")
time.sleep(30)
new_endpoint = sagemaker_session.sagemaker_client.describe_endpoint(
EndpointName=endpoint_name)
current_retry_count += 1
if new_endpoint["EndpointStatus"] == "InService":
break
new_config_name = new_endpoint["EndpointConfigName"]
new_config = sagemaker_session.sagemaker_client.describe_endpoint_config(
EndpointConfigName=new_config_name)
assert old_config_name != new_config_name
assert new_config["ProductionVariants"][0][
"InstanceType"] == cpu_instance_type
assert new_config["ProductionVariants"][0]["InitialInstanceCount"] == 1
def test_deploy_model_with_tags_and_kms(
mxnet_training_job,
sagemaker_session,
mxnet_inference_latest_version,
mxnet_inference_latest_py_version,
cpu_instance_type,
):
endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job
)
model_data = desc["ModelArtifacts"]["S3ModelArtifacts"]
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py")
model = MXNetModel(
model_data,
"SageMakerRole",
entry_point=script_path,
py_version=mxnet_inference_latest_py_version,
sagemaker_session=sagemaker_session,
framework_version=mxnet_inference_latest_version,
)
tags = [{"Key": "TagtestKey", "Value": "TagtestValue"}]
kms_key_arn = get_or_create_kms_key(sagemaker_session)
model.deploy(
1, cpu_instance_type, endpoint_name=endpoint_name, tags=tags, kms_key=kms_key_arn
)
returned_model = sagemaker_session.sagemaker_client.describe_model(ModelName=model.name)
returned_model_tags = sagemaker_session.sagemaker_client.list_tags(
ResourceArn=returned_model["ModelArn"]
)["Tags"]
endpoint = sagemaker_session.sagemaker_client.describe_endpoint(EndpointName=endpoint_name)
endpoint_tags = sagemaker_session.sagemaker_client.list_tags(
ResourceArn=endpoint["EndpointArn"]
)["Tags"]
endpoint_config = sagemaker_session.sagemaker_client.describe_endpoint_config(
EndpointConfigName=endpoint["EndpointConfigName"]
)
endpoint_config_tags = sagemaker_session.sagemaker_client.list_tags(
ResourceArn=endpoint_config["EndpointConfigArn"]
)["Tags"]
production_variants = endpoint_config["ProductionVariants"]
assert returned_model_tags == tags
assert endpoint_config_tags == tags
assert endpoint_tags == tags
assert production_variants[0]["InstanceType"] == cpu_instance_type
assert production_variants[0]["InitialInstanceCount"] == 1
assert endpoint_config["KmsKeyId"] == kms_key_arn
def test_deploy_model_with_tags_and_kms(mxnet_training_job, sagemaker_session,
mxnet_full_version):
endpoint_name = 'test-mxnet-deploy-model-{}'.format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job)
model_data = desc['ModelArtifacts']['S3ModelArtifacts']
script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py')
model = MXNetModel(model_data,
'SageMakerRole',
entry_point=script_path,
py_version=PYTHON_VERSION,
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version)
tags = [{'Key': 'TagtestKey', 'Value': 'TagtestValue'}]
kms_key_arn = get_or_create_kms_key(sagemaker_session)
model.deploy(1,
'ml.m4.xlarge',
endpoint_name=endpoint_name,
tags=tags,
kms_key=kms_key_arn)
returned_model = sagemaker_session.describe_model(
EndpointName=model.name)
returned_model_tags = sagemaker_session.list_tags(
ResourceArn=returned_model['ModelArn'])['Tags']
endpoint = sagemaker_session.describe_endpoint(
EndpointName=endpoint_name)
endpoint_tags = sagemaker_session.list_tags(
ResourceArn=endpoint['EndpointArn'])['Tags']
endpoint_config = sagemaker_session.describe_endpoint_config(
EndpointConfigName=endpoint['EndpointConfigName'])
endpoint_config_tags = sagemaker_session.list_tags(
ResourceArn=endpoint_config['EndpointConfigArn'])['Tags']
production_variants = endpoint_config['ProductionVariants']
assert returned_model_tags == tags
assert endpoint_config_tags == tags
assert endpoint_tags == tags
assert production_variants[0]['InstanceType'] == 'ml.m4.xlarge'
assert production_variants[0]['InitialInstanceCount'] == 1
assert endpoint_config['KmsKeyId'] == kms_key_arn
def test_gluonnlp(sagemaker_session, ecr_image, instance_type,
framework_version):
import urllib.request
tmpdir = tempfile.mkdtemp()
tmpfile = os.path.join(tmpdir, 'bert_sst.tar.gz')
urllib.request.urlretrieve(
'https://aws-dlc-sample-models.s3.amazonaws.com/bert_sst/bert_sst.tar.gz',
tmpfile)
prefix = 'gluonnlp-serving/default-handlers'
model_data = sagemaker_session.upload_data(path=tmpfile, key_prefix=prefix)
model = MXNetModel(model_data,
'SageMakerRole',
SCRIPT_PATH,
image=ecr_image,
py_version="py3",
framework_version=framework_version,
sagemaker_session=sagemaker_session)
endpoint_name = utils.unique_name_from_base('test-mxnet-gluonnlp')
with timeout.timeout_and_delete_endpoint_by_name(endpoint_name,
sagemaker_session):
predictor = model.deploy(1, instance_type, endpoint_name=endpoint_name)
output = predictor.predict(
["Positive sentiment", "Negative sentiment"])
assert [1, 0] == output
def deploy_model(model_data, endpoint_name, sagemaker_session, ecr_image, instance_type, framework_version, role, script="bert_inference.py", source_dir="code/"):
"""
"""
if ecr_image is not None:
model = MXNetModel(model_data,
role,
entry_point=script,
source_dir=source_dir,
image_uri=ecr_image,
py_version="py3",
framework_version=framework_version,
sagemaker_session=sagemaker_session)
else:
model = MXNetModel(model_data,
role,
entry_point=script,
source_dir=source_dir,
py_version="py3",
framework_version=framework_version,
sagemaker_session=sagemaker_session)
endpoint_name = utils.unique_name_from_base(endpoint_name)
print("Deploying...")
predictor = model.deploy(1, instance_type, endpoint_name=endpoint_name)
print("\nEndpoint Name: {}".format(endpoint_name))
return predictor
def test_neuron_hosting(sagemaker_session, ecr_image, instance_type,
framework_version, skip_if_no_neuron):
prefix = "mxnet-serving/neuron-handlers"
model_data = sagemaker_session.upload_data(path=MODEL_PATH,
key_prefix=prefix)
# Since fake inf is run in mnist-neuron.py and NEURONCORE_GROUP_SIZE env variable
# is not set, set the model server worker to 1 (or need to set env variable).
# Otherwise num workers defaults to 4 (4 cpu's) and run out of neuron cores
model = MXNetModel(
model_data,
"SageMakerRole",
SCRIPT_PATH,
framework_version=framework_version,
image_uri=ecr_image,
model_server_workers=1,
sagemaker_session=sagemaker_session,
)
endpoint_name = utils.unique_name_from_base("test-neuron-mxnet-serving")
with timeout.timeout_and_delete_endpoint_by_name(endpoint_name,
sagemaker_session):
predictor = model.deploy(1, instance_type, endpoint_name=endpoint_name)
numpy_ndarray = np.load(INPUT_PATH)
output = predictor.predict(data=numpy_ndarray)
with open(OUTPUT_PATH) as outputfile:
expected_output = json.load(outputfile)
assert expected_output == output
print(output)
def test_neuron_hosting(sagemaker_session, ecr_image, instance_type,
framework_version, skip_if_no_neuron):
prefix = "mxnet-serving/neuron-handlers"
model_data = sagemaker_session.upload_data(path=MODEL_PATH,
key_prefix=prefix)
model = MXNetModel(
model_data,
"SageMakerRole",
SCRIPT_PATH,
framework_version=framework_version,
image_uri=ecr_image,
sagemaker_session=sagemaker_session,
)
endpoint_name = utils.unique_name_from_base("test-neuron-mxnet-serving")
with timeout.timeout_and_delete_endpoint_by_name(endpoint_name,
sagemaker_session):
predictor = model.deploy(1, instance_type, endpoint_name=endpoint_name)
numpy_ndarray = np.load(INPUT_PATH)
output = predictor.predict(data=numpy_ndarray)
with open(OUTPUT_PATH) as outputfile:
expected_output = json.load(outputfile)
assert expected_output == output
print(output)
def test_gluoncv(sagemaker_session, ecr_image, instance_type, framework_version):
try: # python3
from urllib.request import urlretrieve
except: # python2
from urllib import urlretrieve
tmpdir = tempfile.mkdtemp()
tmpfile = os.path.join(tmpdir, 'yolo3_darknet53_voc.tar.gz')
urlretrieve('https://dlc-samples.s3.amazonaws.com/mxnet/gluon/yolo3_darknet53_voc.tar.gz', tmpfile)
prefix = 'gluoncv-serving/default-handlers'
model_data = sagemaker_session.upload_data(path=tmpfile, key_prefix=prefix)
model = MXNetModel(model_data,
'SageMakerRole',
SCRIPT_PATH,
image=ecr_image,
py_version="py3",
framework_version=framework_version,
sagemaker_session=sagemaker_session)
endpoint_name = utils.unique_name_from_base('test-mxnet-gluoncv')
with timeout.timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
predictor = model.deploy(1, instance_type, endpoint_name=endpoint_name)
with open(SCRIPT_DATA_PATH, 'rb') as fdata:
output = predictor.predict([fdata.read().hex()])
assert len(output[0]) == 100
def test_deploy_model(mxnet_training_job, sagemaker_session):
endpoint_name = 'test-mxnet-deploy-model-{}'.format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job)
model_data = desc['ModelArtifacts']['S3ModelArtifacts']
script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist_neo.py')
role = 'SageMakerRole'
model = MXNetModel(model_data,
role,
entry_point=script_path,
py_version=PYTHON_VERSION,
sagemaker_session=sagemaker_session)
model.compile(target_instance_family='ml_m4',
input_shape={'data': [1, 1, 28, 28]},
role=role,
job_name='test-deploy-model-compilation-job-{}'.format(
int(time.time())),
output_path='/'.join(model_data.split('/')[:-1]))
predictor = model.deploy(1,
'ml.m4.xlarge',
endpoint_name=endpoint_name)
predictor.content_type = "application/vnd+python.numpy+binary"
data = numpy.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
def test_deploy_model_with_accelerator(
mxnet_training_job, sagemaker_session, ei_mxnet_full_version
):
endpoint_name = "test-mxnet-deploy-model-ei-{}".format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job
)
model_data = desc["ModelArtifacts"]["S3ModelArtifacts"]
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py")
model = MXNetModel(
model_data,
"SageMakerRole",
entry_point=script_path,
framework_version=ei_mxnet_full_version,
py_version=PYTHON_VERSION,
sagemaker_session=sagemaker_session,
)
predictor = model.deploy(
1, "ml.m4.xlarge", endpoint_name=endpoint_name, accelerator_type="ml.eia1.medium"
)
data = numpy.zeros(shape=(1, 1, 28, 28))
result = predictor.predict(data)
assert result is not None
def test_deploy_model(mxnet_training_job, sagemaker_session, mxnet_full_version):
endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job
)
model_data = desc["ModelArtifacts"]["S3ModelArtifacts"]
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py")
model = MXNetModel(
model_data,
"SageMakerRole",
entry_point=script_path,
py_version=PYTHON_VERSION,
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version,
)
predictor = model.deploy(1, "ml.m4.xlarge", endpoint_name=endpoint_name)
data = numpy.zeros(shape=(1, 1, 28, 28))
result = predictor.predict(data)
assert result is not None
predictor.delete_model()
with pytest.raises(Exception) as exception:
sagemaker_session.sagemaker_client.describe_model(ModelName=model.name)
assert "Could not find model" in str(exception.value)
def test_deploy_model(mxnet_training_job, sagemaker_session, cpu_instance_type,
cpu_instance_family):
endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job)
model_data = desc["ModelArtifacts"]["S3ModelArtifacts"]
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist_neo.py")
role = "SageMakerRole"
model = MXNetModel(
model_data,
role,
entry_point=script_path,
py_version=PYTHON_VERSION,
sagemaker_session=sagemaker_session,
)
model.compile(
target_instance_family=cpu_instance_family,
input_shape={"data": [1, 1, 28, 28]},
role=role,
job_name="test-deploy-model-compilation-job-{}".format(
int(time.time())),
output_path="/".join(model_data.split("/")[:-1]),
)
predictor = model.deploy(1,
cpu_instance_type,
endpoint_name=endpoint_name)
predictor.content_type = "application/vnd+python.numpy+binary"
data = numpy.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
def test_deploy_model(mxnet_training_job, sagemaker_session,
mxnet_full_version):
endpoint_name = 'test-mxnet-deploy-model-{}'.format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job)
model_data = desc['ModelArtifacts']['S3ModelArtifacts']
script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py')
model = MXNetModel(model_data,
'SageMakerRole',
entry_point=script_path,
py_version=PYTHON_VERSION,
sagemaker_session=sagemaker_session,
framework_version=mxnet_full_version)
predictor = model.deploy(1,
'ml.m4.xlarge',
endpoint_name=endpoint_name)
data = numpy.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
predictor.delete_model()
with pytest.raises(Exception) as exception:
sagemaker_session.sagemaker_client.describe_model(ModelName=model.name)
assert 'Could not find model' in str(exception.value)
def test_inferentia_deploy_model(mxnet_training_job, sagemaker_session,
inf_instance_type, inf_instance_family):
endpoint_name = unique_name_from_base("test-neo-deploy-model")
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job)
model_data = desc["ModelArtifacts"]["S3ModelArtifacts"]
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist_neo.py")
role = "SageMakerRole"
model = MXNetModel(
model_data,
role,
entry_point=script_path,
framework_version=INF_MXNET_VERSION,
sagemaker_session=sagemaker_session,
)
model.compile(
target_instance_family=inf_instance_family,
input_shape={"data": [1, 1, 28, 28]},
role=role,
job_name=unique_name_from_base(
"test-deploy-model-compilation-job"),
output_path="/".join(model_data.split("/")[:-1]),
)
predictor = model.deploy(1,
inf_instance_type,
endpoint_name=endpoint_name)
predictor.content_type = "application/vnd+python.numpy+binary"
data = numpy.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
def test_git_support_codecommit_with_mxnet(sagemaker_local_session):
script_path = "mnist.py"
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
git_config = {
"repo": CODECOMMIT_REPO,
"branch": CODECOMMIT_BRANCH,
"username": "******",
"password": "******",
}
source_dir = "mxnet"
dependencies = ["foo/bar.py"]
mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
source_dir=source_dir,
dependencies=dependencies,
framework_version=MXNet.LATEST_VERSION,
py_version=PYTHON_VERSION,
train_instance_count=1,
train_instance_type="local",
sagemaker_session=sagemaker_local_session,
git_config=git_config,
)
mx.fit(
{
"train": "file://" + os.path.join(data_path, "train"),
"test": "file://" + os.path.join(data_path, "test"),
}
)
files = [file for file in os.listdir(mx.source_dir)]
assert "some_file" in files
assert "mnist.py" in files
assert os.path.exists(mx.dependencies[0])
with lock.lock(LOCK_PATH):
try:
client = sagemaker_local_session.sagemaker_client
desc = client.describe_training_job(TrainingJobName=mx.latest_training_job.name)
model_data = desc["ModelArtifacts"]["S3ModelArtifacts"]
model = MXNetModel(
model_data,
"SageMakerRole",
entry_point=script_path,
source_dir=source_dir,
dependencies=dependencies,
py_version=PYTHON_VERSION,
sagemaker_session=sagemaker_local_session,
framework_version=MXNet.LATEST_VERSION,
git_config=git_config,
)
predictor = model.deploy(1, "local")
data = numpy.zeros(shape=(1, 1, 28, 28))
result = predictor.predict(data)
assert result is not None
finally:
predictor.delete_endpoint()
def test_deploy_model(mxnet_training_job, sagemaker_session):
endpoint_name = 'test-mxnet-deploy-model-{}'.format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(TrainingJobName=mxnet_training_job)
model_data = desc['ModelArtifacts']['S3ModelArtifacts']
script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py')
model = MXNetModel(model_data, 'SageMakerRole', entry_point=script_path, sagemaker_session=sagemaker_session)
predictor = model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name)
data = numpy.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
def predictor(docker_image, sagemaker_local_session, local_instance_type):
default_handler_path = os.path.join(RESOURCE_PATH, 'default_handlers')
m = MXNetModel('file://{}'.format(os.path.join(default_handler_path, 'model')), 'SageMakerRole',
os.path.join(default_handler_path, 'code', 'empty_module.py'),
image=docker_image, sagemaker_session=sagemaker_local_session,
model_server_workers=NUM_MODEL_SERVER_WORKERS)
with local_mode_utils.lock():
try:
predictor = m.deploy(1, local_instance_type)
yield predictor
finally:
sagemaker_local_session.delete_endpoint(m.endpoint_name)
def test_deploy_model(mxnet_training_job, sagemaker_session):
endpoint_name = 'test-mxnet-deploy-model-{}'.format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, minutes=20):
desc = sagemaker_session.sagemaker_client.describe_training_job(TrainingJobName=mxnet_training_job)
model_data = desc['ModelArtifacts']['S3ModelArtifacts']
script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py')
model = MXNetModel(model_data, 'SageMakerRole', entry_point=script_path, sagemaker_session=sagemaker_session)
predictor = model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name)
data = numpy.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
def predictor(docker_image, sagemaker_local_session, local_instance_type):
model = MXNetModel('file://{}'.format(MODEL_PATH),
'SageMakerRole',
SCRIPT_PATH,
image=docker_image,
sagemaker_session=sagemaker_local_session)
with local_mode_utils.lock():
try:
predictor = model.deploy(1, local_instance_type)
yield predictor
finally:
predictor.delete_endpoint()
def test_deploy_model_with_accelerator(mxnet_training_job, sagemaker_session, ei_mxnet_full_version):
endpoint_name = 'test-mxnet-deploy-model-ei-{}'.format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(TrainingJobName=mxnet_training_job)
model_data = desc['ModelArtifacts']['S3ModelArtifacts']
script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'mnist.py')
model = MXNetModel(model_data, 'SageMakerRole', entry_point=script_path,
framework_version=ei_mxnet_full_version, py_version=PYTHON_VERSION,
sagemaker_session=sagemaker_session)
predictor = model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name, accelerator_type='ml.eia1.medium')
data = numpy.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
def test_hosting(docker_image, sagemaker_local_session, local_instance_type):
model = MXNetModel('file://{}'.format(MODEL_PATH),
'SageMakerRole',
SCRIPT_PATH,
image=docker_image,
sagemaker_session=sagemaker_local_session)
input = json.dumps({'some': 'json'})
with local_mode_utils.lock():
try:
predictor = model.deploy(1, local_instance_type)
output = predictor.predict(input)
assert input == output
finally:
sagemaker_local_session.delete_endpoint(model.endpoint_name)
def test_hosting(docker_image, sagemaker_local_session, local_instance_type):
hosting_resource_path = os.path.join(RESOURCE_PATH, 'dummy_hosting')
m = MXNetModel('file://{}'.format(os.path.join(hosting_resource_path, 'code')), 'SageMakerRole',
os.path.join(hosting_resource_path, 'code', 'dummy_hosting_module.py'),
image=docker_image, sagemaker_session=sagemaker_local_session,
model_server_workers=NUM_MODEL_SERVER_WORKERS)
input = json.dumps({'some': 'json'})
with local_mode_utils.lock():
try:
predictor = m.deploy(1, local_instance_type)
output = predictor.predict(input)
assert input == output
finally:
sagemaker_local_session.delete_endpoint(m.endpoint_name)
def _test_hosting_function(ecr_image, sagemaker_session, instance_type, framework_version):
prefix = 'mxnet-serving/default-handlers'
model_data = sagemaker_session.upload_data(path=MODEL_PATH, key_prefix=prefix)
model = MXNetModel(model_data,
'SageMakerRole',
SCRIPT_PATH,
framework_version=framework_version,
image_uri=ecr_image,
sagemaker_session=sagemaker_session)
endpoint_name = utils.unique_name_from_base('test-mxnet-serving')
with timeout.timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
predictor = model.deploy(1, instance_type, endpoint_name=endpoint_name)
output = predictor.predict([[1, 2]])
assert [[4.9999918937683105]] == output
def test_deploy_model(
mxnet_training_job,
sagemaker_session,
cpu_instance_type,
cpu_instance_family,
neo_mxnet_latest_version,
neo_mxnet_latest_py_version,
):
endpoint_name = unique_name_from_base("test-neo-deploy-model")
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job)
model_data = desc["ModelArtifacts"]["S3ModelArtifacts"]
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist_neo.py")
role = "SageMakerRole"
model = MXNetModel(
model_data,
role,
entry_point=script_path,
py_version=neo_mxnet_latest_py_version,
framework_version=neo_mxnet_latest_version,
sagemaker_session=sagemaker_session,
)
serializer = JSONSerializer()
serializer.CONTENT_TYPE = "application/vnd+python.numpy+binary"
model.compile(
target_instance_family=cpu_instance_family,
input_shape={
"data": [1, 1, 28, 28],
"softmax_label": [1]
},
role=role,
job_name=unique_name_from_base(
"test-deploy-model-compilation-job"),
output_path="/".join(model_data.split("/")[:-1]),
)
predictor = model.deploy(1,
cpu_instance_type,
serializer=serializer,
endpoint_name=endpoint_name)
data = numpy.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
def test_gluon_hosting(docker_image, sagemaker_local_session, local_instance_type):
model = MXNetModel('file://{}'.format(MODEL_PATH),
'SageMakerRole',
SCRIPT_PATH,
image=docker_image,
sagemaker_session=sagemaker_local_session)
with open(os.path.join(RESOURCE_PATH, 'mnist', 'images', '04.json'), 'r') as f:
input = json.load(f)
with local_mode_utils.lock():
try:
predictor = model.deploy(1, local_instance_type)
output = predictor.predict(input)
assert [4.0] == output
finally:
predictor.delete_endpoint()
def test_register_model_package_versioned(
mxnet_training_job,
sagemaker_session,
mxnet_inference_latest_version,
mxnet_inference_latest_py_version,
cpu_instance_type,
):
endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job)
model_package_group_name = "register-model-package-{}".format(
sagemaker_timestamp())
sagemaker_session.sagemaker_client.create_model_package_group(
ModelPackageGroupName=model_package_group_name)
model_data = desc["ModelArtifacts"]["S3ModelArtifacts"]
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py")
model = MXNetModel(
model_data,
"SageMakerRole",
entry_point=script_path,
py_version=mxnet_inference_latest_py_version,
sagemaker_session=sagemaker_session,
framework_version=mxnet_inference_latest_version,
)
model_pkg = model.register(
content_types=["application/json"],
response_types=["application/json"],
inference_instances=["ml.m5.large"],
transform_instances=["ml.m5.large"],
model_package_group_name=model_package_group_name,
approval_status="Approved",
)
assert isinstance(model_pkg, ModelPackage)
predictor = model.deploy(1,
cpu_instance_type,
endpoint_name=endpoint_name)
data = numpy.zeros(shape=(1, 1, 28, 28))
result = predictor.predict(data)
assert result is not None
sagemaker_session.sagemaker_client.delete_model_package(
ModelPackageName=model_pkg.model_package_arn)
sagemaker_session.sagemaker_client.delete_model_package_group(
ModelPackageGroupName=model_package_group_name)
def test_deploy_model_and_update_endpoint(
mxnet_training_job,
sagemaker_session,
mxnet_inference_latest_version,
mxnet_inference_latest_py_version,
cpu_instance_type,
alternative_cpu_instance_type,
):
endpoint_name = "test-mxnet-deploy-model-{}".format(sagemaker_timestamp())
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
desc = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=mxnet_training_job)
model_data = desc["ModelArtifacts"]["S3ModelArtifacts"]
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py")
model = MXNetModel(
model_data,
"SageMakerRole",
entry_point=script_path,
py_version=mxnet_inference_latest_py_version,
sagemaker_session=sagemaker_session,
framework_version=mxnet_inference_latest_version,
)
predictor = model.deploy(1,
alternative_cpu_instance_type,
endpoint_name=endpoint_name)
endpoint_desc = sagemaker_session.sagemaker_client.describe_endpoint(
EndpointName=endpoint_name)
old_config_name = endpoint_desc["EndpointConfigName"]
predictor.update_endpoint(initial_instance_count=1,
instance_type=cpu_instance_type)
endpoint_desc = sagemaker_session.sagemaker_client.describe_endpoint(
EndpointName=endpoint_name)
new_config_name = endpoint_desc["EndpointConfigName"]
new_config = sagemaker_session.sagemaker_client.describe_endpoint_config(
EndpointConfigName=new_config_name)
assert old_config_name != new_config_name
assert new_config["ProductionVariants"][0][
"InstanceType"] == cpu_instance_type
assert new_config["ProductionVariants"][0]["InitialInstanceCount"] == 1