def test_pipe_mode(instance_type, sagemaker_session, docker_image_uri):
framework_version = docker_image_uri.split(':')[-1].split('-')[0]
if framework_version not in PIPE_MODE_VERSIONS:
pytest.skip('skipping non-pipe-mode version {} because it is not in {}'
.format(framework_version, PIPE_MODE_VERSIONS))
script_path = 'test/resources/synthetic'
with timeout(minutes=15):
estimator = MyEstimator(entry_point='synthetic_pipe_mode_dataset.py',
source_dir=script_path,
role='SageMakerRole',
input_mode='Pipe',
training_steps=100,
evaluation_steps=10,
train_instance_count=1,
train_instance_type=instance_type,
sagemaker_session=sagemaker_session,
docker_image_uri=docker_image_uri)
logger.info("uploading training data")
train_data = 's3://sagemaker-sample-data-us-west-2/tensorflow/pipe-mode/train'
eval_data = 's3://sagemaker-sample-data-us-west-2/tensorflow/pipe-mode/eval'
logger.info("fitting estimator")
estimator.fit({'train': train_data, 'eval': eval_data})
def test_distributed(instance_type, sagemaker_session, docker_image_uri):
with timeout(minutes=15):
estimator = MyEstimator(entry_point='resnet_cifar_10.py',
source_dir=script_path,
role='SageMakerRole',
training_steps=2,
evaluation_steps=1,
train_instance_count=2,
train_instance_type=instance_type,
sagemaker_session=sagemaker_session,
docker_image_uri=docker_image_uri)
logger.info("uploading training data")
key_prefix = 'integ-test-data/tf-cifar-{}'.format(instance_type)
inputs = estimator.sagemaker_session.upload_data(path=data_path,
key_prefix=key_prefix)
logger.info("fitting estimator")
estimator.fit(inputs)
with timeout_and_delete_endpoint(estimator=estimator, minutes=30):
logger.info("deploy model")
json_predictor = estimator.deploy(initial_instance_count=1,
instance_type=instance_type)
data = np.random.rand(32, 32, 3)
predict_response = json_predictor.predict(data)
assert len(
predict_response['outputs']['probabilities']['floatVal']) == 10
def test_distributed(instance_type, sagemaker_session, docker_image_uri):
script_path = 'test/resources/cifar_10/code'
data_path = 'test/resources/cifar_10/data/training'
with timeout(minutes=15):
estimator = MyEstimator(entry_point='resnet_cifar_10.py',
source_dir=script_path,
role='SageMakerRole',
training_steps=500,
evaluation_steps=1,
train_instance_count=2,
train_instance_type=instance_type,
sagemaker_session=sagemaker_session,
docker_image_uri=docker_image_uri)
logger.info("uploading training data")
key_prefix = 'integ-test-data/tf-cifar-{}'.format(instance_type)
inputs = estimator.sagemaker_session.upload_data(path=data_path,
key_prefix=key_prefix)
logger.info("fitting estimator")
estimator.fit(inputs)
with timeout_and_delete_endpoint(estimator=estimator, minutes=30):
logger.info("deploy model")
json_predictor = estimator.deploy(initial_instance_count=1, instance_type=instance_type)
data = np.random.rand(32, 32, 3)
predict_response = json_predictor.predict(data)
assert len(predict_response['outputs']['probabilities']['floatVal']) == 10
def test_pipe_mode(instance_type, sagemaker_session, docker_image_uri):
framework_version = docker_image_uri.split(':')[-1].split('-')[0]
if framework_version not in PIPE_MODE_VERSIONS:
pytest.skip(
'skipping non-pipe-mode version {} because it is not in {}'.format(
framework_version, PIPE_MODE_VERSIONS))
script_path = 'test/resources/synthetic'
with timeout(minutes=15):
estimator = MyEstimator(entry_point='synthetic_pipe_mode_dataset.py',
source_dir=script_path,
role='SageMakerRole',
input_mode='Pipe',
training_steps=100,
evaluation_steps=10,
train_instance_count=1,
train_instance_type=instance_type,
sagemaker_session=sagemaker_session,
docker_image_uri=docker_image_uri)
logger.info("uploading training data")
train_data = 's3://sagemaker-sample-data-us-west-2/tensorflow/pipe-mode/train'
eval_data = 's3://sagemaker-sample-data-us-west-2/tensorflow/pipe-mode/eval'
logger.info("fitting estimator")
estimator.fit({'train': train_data, 'eval': eval_data})