def test_mxnet_local_data_local_script(mxnet_training_latest_version,
mxnet_training_latest_py_version):
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
script_path = os.path.join(data_path, "mnist.py")
mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
instance_count=1,
instance_type="local",
framework_version=mxnet_training_latest_version,
py_version=mxnet_training_latest_py_version,
sagemaker_session=LocalNoS3Session(),
)
train_input = "file://" + os.path.join(data_path, "train")
test_input = "file://" + os.path.join(data_path, "test")
mx.fit({"train": train_input, "test": test_input})
endpoint_name = mx.latest_training_job.name
with lock.lock(LOCK_PATH):
try:
predictor = mx.deploy(1, "local", endpoint_name=endpoint_name)
data = numpy.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
finally:
predictor.delete_endpoint()
def test_source_dirs(tmpdir, sagemaker_local_session):
source_dir = os.path.join(DATA_DIR, "pytorch_source_dirs")
lib = os.path.join(str(tmpdir), "alexa.py")
with open(lib, "w") as f:
f.write("def question(to_anything): return 42")
estimator = PyTorch(
entry_point="train.py",
role="SageMakerRole",
source_dir=source_dir,
dependencies=[lib],
py_version=PYTHON_VERSION,
train_instance_count=1,
train_instance_type="local",
sagemaker_session=sagemaker_local_session,
)
estimator.fit()
# endpoint tests all use the same port, so we use this lock to prevent concurrent execution
with lock.lock():
try:
predictor = estimator.deploy(initial_instance_count=1, instance_type="local")
predict_response = predictor.predict([7])
assert predict_response == [49]
finally:
estimator.delete_endpoint()
def test_github(sagemaker_local_session, pytorch_inference_latest_version,
pytorch_inference_latest_py_version):
script_path = "mnist.py"
git_config = {"repo": GIT_REPO, "branch": BRANCH, "commit": COMMIT}
pytorch = PyTorch(
entry_point=script_path,
role="SageMakerRole",
source_dir="pytorch",
framework_version=pytorch_inference_latest_version,
py_version=pytorch_inference_latest_py_version,
instance_count=1,
instance_type="local",
sagemaker_session=sagemaker_local_session,
git_config=git_config,
)
data_path = os.path.join(DATA_DIR, "pytorch_mnist")
pytorch.fit({"training": "file://" + os.path.join(data_path, "training")})
with lock.lock(LOCK_PATH):
try:
predictor = pytorch.deploy(initial_instance_count=1,
instance_type="local")
data = numpy.zeros(shape=(1, 1, 28, 28)).astype(numpy.float32)
result = predictor.predict(data)
assert 10 == len(
result[0]) # check that there is a probability for each label
finally:
predictor.delete_endpoint()
def test_source_dirs(tmpdir, sagemaker_local_session):
source_dir = os.path.join(DATA_DIR, "pytorch_source_dirs")
lib = os.path.join(str(tmpdir), "alexa.py")
with open(lib, "w") as f:
f.write("def question(to_anything): return 42")
# TODO: fails on newer versions of pytorch in call to np.load(BytesIO(stream.read()))
# "ValueError: Cannot load file containing pickled data when allow_pickle=False"
estimator = PyTorch(
entry_point="train.py",
role="SageMakerRole",
source_dir=source_dir,
dependencies=[lib],
framework_version=
"0.4", # hard-code to last known good pytorch for now (see TODO above)
py_version="py3",
instance_count=1,
instance_type="local",
sagemaker_session=sagemaker_local_session,
)
estimator.fit()
# endpoint tests all use the same port, so we use this lock to prevent concurrent execution
with lock.lock():
try:
predictor = estimator.deploy(initial_instance_count=1,
instance_type="local")
predict_response = predictor.predict([7])
assert predict_response == [49]
finally:
predictor.delete_endpoint()
def test_mxnet_local_mode(sagemaker_local_session,
mxnet_training_latest_version,
mxnet_training_latest_py_version):
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py")
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
py_version=mxnet_training_latest_py_version,
instance_count=1,
instance_type="local",
sagemaker_session=sagemaker_local_session,
framework_version=mxnet_training_latest_version,
)
train_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, "train"),
key_prefix="integ-test-data/mxnet_mnist/train")
test_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, "test"),
key_prefix="integ-test-data/mxnet_mnist/test")
mx.fit({"train": train_input, "test": test_input})
endpoint_name = mx.latest_training_job.name
with lock.lock(LOCK_PATH):
try:
predictor = mx.deploy(1, "local", endpoint_name=endpoint_name)
data = numpy.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
finally:
predictor.delete_endpoint()
def test_github(sagemaker_local_session):
script_path = "mnist.py"
data_path = os.path.join(DATA_DIR, "pytorch_mnist")
git_config = {"repo": GIT_REPO, "branch": BRANCH, "commit": COMMIT}
pytorch = PyTorch(
entry_point=script_path,
role="SageMakerRole",
source_dir="pytorch",
framework_version=PYTORCH_VERSION,
py_version=PYTHON_VERSION,
train_instance_count=1,
train_instance_type="local",
sagemaker_session=sagemaker_local_session,
git_config=git_config,
)
pytorch.fit({
"training":
"file://" + os.path.join(data_path, "training", MNIST_FOLDER_NAME)
})
with lock.lock(LOCK_PATH):
try:
predictor = pytorch.deploy(initial_instance_count=1,
instance_type="local")
data = numpy.zeros(shape=(1, 1, 28, 28)).astype(numpy.float32)
result = predictor.predict(data)
assert result is not None
finally:
predictor.delete_endpoint()
def test_git_support_codecommit_with_mxnet(sagemaker_local_session):
script_path = "mnist.py"
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
git_config = {
"repo": CODECOMMIT_REPO,
"branch": CODECOMMIT_BRANCH,
"username": "******",
"password": "******",
}
source_dir = "mxnet"
dependencies = ["foo/bar.py"]
mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
source_dir=source_dir,
dependencies=dependencies,
framework_version=MXNet.LATEST_VERSION,
py_version=PYTHON_VERSION,
train_instance_count=1,
train_instance_type="local",
sagemaker_session=sagemaker_local_session,
git_config=git_config,
)
mx.fit(
{
"train": "file://" + os.path.join(data_path, "train"),
"test": "file://" + os.path.join(data_path, "test"),
}
)
files = [file for file in os.listdir(mx.source_dir)]
assert "some_file" in files
assert "mnist.py" in files
assert os.path.exists(mx.dependencies[0])
with lock.lock(LOCK_PATH):
try:
client = sagemaker_local_session.sagemaker_client
desc = client.describe_training_job(TrainingJobName=mx.latest_training_job.name)
model_data = desc["ModelArtifacts"]["S3ModelArtifacts"]
model = MXNetModel(
model_data,
"SageMakerRole",
entry_point=script_path,
source_dir=source_dir,
dependencies=dependencies,
py_version=PYTHON_VERSION,
sagemaker_session=sagemaker_local_session,
framework_version=MXNet.LATEST_VERSION,
git_config=git_config,
)
predictor = model.deploy(1, "local")
data = numpy.zeros(shape=(1, 1, 28, 28))
result = predictor.predict(data)
assert result is not None
finally:
predictor.delete_endpoint()
def test_private_github_with_2fa(sagemaker_local_session,
sklearn_latest_version,
sklearn_latest_py_version):
script_path = "mnist.py"
data_path = os.path.join(DATA_DIR, "sklearn_mnist")
git_config = {
"repo": PRIVATE_GIT_REPO_2FA,
"branch": PRIVATE_BRANCH_2FA,
"commit": PRIVATE_COMMIT_2FA,
"2FA_enabled": True,
"token": "", # TODO: find a secure approach
}
source_dir = "sklearn"
sklearn = SKLearn(
entry_point=script_path,
role="SageMakerRole",
source_dir=source_dir,
py_version=sklearn_latest_py_version,
instance_count=1,
instance_type="local",
sagemaker_session=sagemaker_local_session,
framework_version=sklearn_latest_version,
hyperparameters={"epochs": 1},
git_config=git_config,
)
train_input = "file://" + os.path.join(data_path, "train")
test_input = "file://" + os.path.join(data_path, "test")
sklearn.fit({"train": train_input, "test": test_input})
assert os.path.isdir(sklearn.source_dir)
with lock.lock(LOCK_PATH):
try:
client = sagemaker_local_session.sagemaker_client
desc = client.describe_training_job(
TrainingJobName=sklearn.latest_training_job.name)
model_data = desc["ModelArtifacts"]["S3ModelArtifacts"]
model = SKLearnModel(
model_data,
"SageMakerRole",
entry_point=script_path,
framework_version=sklearn_latest_version,
source_dir=source_dir,
sagemaker_session=sagemaker_local_session,
git_config=git_config,
)
predictor = model.deploy(1, "local")
data = numpy.zeros((100, 784), dtype="float32")
result = predictor.predict(data)
assert result is not None
finally:
predictor.delete_endpoint()
def get_or_create_vpc_resources(ec2_client):
# use lock to prevent race condition when tests are running concurrently
with lock.lock(LOCK_PATH):
if _vpc_exists(ec2_client, VPC_NAME):
print("using existing vpc: {}".format(VPC_NAME))
return (
_get_subnet_ids_by_name(ec2_client, VPC_NAME),
_get_security_id_by_name(ec2_client, VPC_NAME),
)
else:
print("creating new vpc: {}".format(VPC_NAME))
return _create_vpc_with_name(ec2_client, VPC_NAME)
def test_local_transform_mxnet(
sagemaker_local_session,
tmpdir,
mxnet_inference_latest_version,
mxnet_inference_latest_py_version,
cpu_instance_type,
):
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
script_path = os.path.join(data_path, "mnist.py")
mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
instance_count=1,
instance_type="local",
framework_version=mxnet_inference_latest_version,
py_version=mxnet_inference_latest_py_version,
sagemaker_session=sagemaker_local_session,
)
train_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, "train"),
key_prefix="integ-test-data/mxnet_mnist/train")
test_input = mx.sagemaker_session.upload_data(
path=os.path.join(data_path, "test"),
key_prefix="integ-test-data/mxnet_mnist/test")
with stopit.ThreadingTimeout(5 * 60, swallow_exc=False):
mx.fit({"train": train_input, "test": test_input})
transform_input_path = os.path.join(data_path, "transform")
transform_input_key_prefix = "integ-test-data/mxnet_mnist/transform"
transform_input = mx.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix)
output_path = "file://%s" % (str(tmpdir))
transformer = mx.transformer(
1,
"local",
assemble_with="Line",
max_payload=1,
strategy="SingleRecord",
output_path=output_path,
)
with lock.lock(LOCK_PATH):
transformer.transform(transform_input,
content_type="text/csv",
split_type="Line")
transformer.wait()
assert os.path.exists(os.path.join(str(tmpdir), "data.csv.out"))
def check_or_create_vpc_resources_efs_fsx(sagemaker_session, name=VPC_NAME):
# use lock to prevent race condition when tests are running concurrently
with lock.lock(LOCK_PATH_EFS):
ec2_client = sagemaker_session.boto_session.client("ec2")
if _vpc_exists(ec2_client, name):
vpc_id = _vpc_id_by_name(ec2_client, name)
return (
_get_subnet_ids_by_name(ec2_client, name),
_security_group_ids_by_vpc_id(sagemaker_session, vpc_id),
)
else:
return _create_vpc_with_name_efs_fsx(ec2_client, name)
def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model, mxnet_full_version):
path = "s3://%s" % sagemaker_local_session.default_bucket()
s3_model = mxnet_model(path)
s3_model.sagemaker_session = sagemaker_local_session
predictor = None
with lock.lock(LOCK_PATH):
try:
predictor = s3_model.deploy(initial_instance_count=1, instance_type="local")
data = numpy.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
finally:
if predictor:
predictor.delete_endpoint()
def test_local_mode_serving_from_local_model(tmpdir, sagemaker_local_session, mxnet_model):
predictor = None
with lock.lock(LOCK_PATH):
try:
path = "file://%s" % (str(tmpdir))
model = mxnet_model(path)
model.sagemaker_session = sagemaker_local_session
predictor = model.deploy(initial_instance_count=1, instance_type="local")
data = numpy.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
finally:
if predictor:
predictor.delete_endpoint()
def test_private_github(
sagemaker_local_session, mxnet_training_latest_version, mxnet_training_latest_py_version
):
script_path = "mnist.py"
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
git_config = {
"repo": PRIVATE_GIT_REPO,
"branch": PRIVATE_BRANCH,
"commit": PRIVATE_COMMIT,
"2FA_enabled": False,
"username": "******",
"password": "", # TODO: find a secure approach
}
source_dir = "mxnet"
dependencies = ["foo/bar.py"]
mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
source_dir=source_dir,
dependencies=dependencies,
framework_version=mxnet_training_latest_version,
py_version=mxnet_training_latest_py_version,
instance_count=1,
instance_type="local",
sagemaker_session=sagemaker_local_session,
git_config=git_config,
)
mx.fit(
{
"train": "file://" + os.path.join(data_path, "train"),
"test": "file://" + os.path.join(data_path, "test"),
}
)
files = [file for file in os.listdir(mx.source_dir)]
assert "some_file" in files
assert "mnist.py" in files
assert os.path.exists(mx.dependencies[0])
with lock.lock(LOCK_PATH):
try:
serving_script_path = "mnist_hosting_with_custom_handlers.py"
predictor = mx.deploy(1, "local", entry_point=serving_script_path)
data = numpy.zeros(shape=(1, 1, 28, 28))
result = predictor.predict(data)
assert result is not None
finally:
predictor.delete_endpoint()
def test_codecommit(
sagemaker_local_session, mxnet_training_latest_version, mxnet_training_latest_py_version
):
script_path = "mnist.py"
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
git_config = {
"repo": CODECOMMIT_REPO,
"branch": CODECOMMIT_BRANCH,
"username": "******",
"password": "", # TODO: assume a role to get temporary credentials
}
source_dir = "mxnet"
dependencies = ["foo/bar.py"]
mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
source_dir=source_dir,
dependencies=dependencies,
framework_version=mxnet_training_latest_version,
py_version=mxnet_training_latest_py_version,
instance_count=1,
instance_type="local",
sagemaker_session=sagemaker_local_session,
git_config=git_config,
)
mx.fit(
{
"train": "file://" + os.path.join(data_path, "train"),
"test": "file://" + os.path.join(data_path, "test"),
}
)
files = [file for file in os.listdir(mx.source_dir)]
assert "some_file" in files
assert "mnist.py" in files
assert os.path.exists(mx.dependencies[0])
with lock.lock(LOCK_PATH):
try:
predictor = mx.deploy(1, "local")
data = numpy.zeros(shape=(1, 1, 28, 28))
result = predictor.predict(data)
assert result is not None
finally:
predictor.delete_endpoint()
def test_mxnet_local_data_local_script(mxnet_training_latest_version,
mxnet_training_latest_py_version):
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
script_path = os.path.join(data_path, "mnist.py")
local_no_s3_session = LocalNoS3Session()
local_no_s3_session.boto_session.resource = Mock(
side_effect=local_no_s3_session.boto_session.resource)
local_no_s3_session.boto_session.client = Mock(
side_effect=local_no_s3_session.boto_session.client)
mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
instance_count=1,
instance_type="local",
framework_version=mxnet_training_latest_version,
py_version=mxnet_training_latest_py_version,
sagemaker_session=local_no_s3_session,
)
train_input = "file://" + os.path.join(data_path, "train")
test_input = "file://" + os.path.join(data_path, "test")
mx.fit({"train": train_input, "test": test_input})
endpoint_name = mx.latest_training_job.name
with lock.lock(LOCK_PATH):
try:
predictor = mx.deploy(1, "local", endpoint_name=endpoint_name)
data = numpy.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
# check if no boto_session s3 calls were made
with pytest.raises(AssertionError):
local_no_s3_session.boto_session.resource.assert_called_with(
"s3", region_name=ANY)
with pytest.raises(AssertionError):
local_no_s3_session.boto_session.client.assert_called_with(
"s3", region_name=ANY)
finally:
predictor.delete_endpoint()
def test_tf_local_mode(sagemaker_local_session):
with stopit.ThreadingTimeout(5 * 60, swallow_exc=False):
script_path = os.path.join(DATA_DIR, "iris", "iris-dnn-classifier.py")
estimator = TensorFlow(
entry_point=script_path,
role="SageMakerRole",
framework_version="1.12",
training_steps=1,
evaluation_steps=1,
hyperparameters={"input_tensor_name": "inputs"},
train_instance_count=1,
train_instance_type="local",
base_job_name="test-tf",
sagemaker_session=sagemaker_local_session,
)
inputs = estimator.sagemaker_session.upload_data(
path=DATA_PATH, key_prefix="integ-test-data/tf_iris"
)
estimator.fit(inputs)
print("job succeeded: {}".format(estimator.latest_training_job.name))
endpoint_name = estimator.latest_training_job.name
with lock.lock(LOCK_PATH):
try:
json_predictor = estimator.deploy(
initial_instance_count=1, instance_type="local", endpoint_name=endpoint_name
)
features = [6.4, 3.2, 4.5, 1.5]
dict_result = json_predictor.predict({"inputs": features})
print("predict result: {}".format(dict_result))
list_result = json_predictor.predict(features)
print("predict result: {}".format(list_result))
assert dict_result == list_result
finally:
estimator.delete_endpoint()
def test_git_support_with_mxnet(sagemaker_local_session, mxnet_full_version):
script_path = "mnist.py"
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
git_config = {"repo": GIT_REPO, "branch": BRANCH, "commit": COMMIT}
dependencies = ["foo/bar.py"]
mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
source_dir="mxnet",
dependencies=dependencies,
framework_version=MXNet.LATEST_VERSION,
py_version=PYTHON_VERSION,
train_instance_count=1,
train_instance_type="local",
sagemaker_session=sagemaker_local_session,
git_config=git_config,
)
mx.fit({
"train": "file://" + os.path.join(data_path, "train"),
"test": "file://" + os.path.join(data_path, "test"),
})
files = [file for file in os.listdir(mx.source_dir)]
assert "some_file" in files
assert "mnist.py" in files
assert os.path.exists(mx.dependencies[0])
with lock.lock(LOCK_PATH):
try:
predictor = mx.deploy(initial_instance_count=1,
instance_type="local")
data = numpy.zeros(shape=(1, 1, 28, 28))
result = predictor.predict(data)
assert result is not None
finally:
predictor.delete_endpoint()
def test_git_support_with_mxnet(sagemaker_local_session):
script_path = "mnist.py"
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
git_config = {
"repo": PRIVATE_GIT_REPO,
"branch": PRIVATE_BRANCH,
"commit": PRIVATE_COMMIT,
"2FA_enabled": False,
"username": "******",
"password": "******",
}
source_dir = "mxnet"
dependencies = ["foo/bar.py"]
mx = MXNet(
entry_point=script_path,
role="SageMakerRole",
source_dir=source_dir,
dependencies=dependencies,
framework_version=MXNet.LATEST_VERSION,
py_version=PYTHON_VERSION,
train_instance_count=1,
train_instance_type="local",
sagemaker_session=sagemaker_local_session,
git_config=git_config,
)
mx.fit({
"train": "file://" + os.path.join(data_path, "train"),
"test": "file://" + os.path.join(data_path, "test"),
})
files = [file for file in os.listdir(mx.source_dir)]
assert "some_file" in files
assert "mnist.py" in files
assert os.path.exists(mx.dependencies[0])
with lock.lock(LOCK_PATH):
try:
serving_script_path = "mnist_hosting_with_custom_handlers.py"
client = sagemaker_local_session.sagemaker_client
desc = client.describe_training_job(
TrainingJobName=mx.latest_training_job.name)
model_data = desc["ModelArtifacts"]["S3ModelArtifacts"]
model = MXNetModel(
model_data,
"SageMakerRole",
entry_point=serving_script_path,
source_dir=source_dir,
dependencies=dependencies,
py_version=PYTHON_VERSION,
sagemaker_session=sagemaker_local_session,
framework_version=MXNet.LATEST_VERSION,
git_config=git_config,
)
predictor = model.deploy(initial_instance_count=1,
instance_type="local")
data = numpy.zeros(shape=(1, 1, 28, 28))
result = predictor.predict(data)
assert result is not None
finally:
predictor.delete_endpoint()
