def test_tuning_mxnet(sagemaker_session):
with timeout(minutes=15):
script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'tuning.py')
data_path = os.path.join(DATA_DIR, 'mxnet_mnist')
estimator = MXNet(entry_point=script_path,
role='SageMakerRole',
train_instance_count=1,
train_instance_type='ml.m4.xlarge',
sagemaker_session=sagemaker_session,
base_job_name='tune-mxnet')
hyperparameter_ranges = {'learning_rate': ContinuousParameter(0.01, 0.2)}
objective_metric_name = 'Validation-accuracy'
metric_definitions = [{'Name': 'Validation-accuracy', 'Regex': 'Validation-accuracy=([0-9\\.]+)'}]
tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions,
max_jobs=4, max_parallel_jobs=2)
train_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'train'),
key_prefix='integ-test-data/mxnet_mnist/train')
test_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'test'),
key_prefix='integ-test-data/mxnet_mnist/test')
tuner.fit({'train': train_input, 'test': test_input})
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
data = np.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
def test_tuning_mxnet(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'tuning.py')
data_path = os.path.join(DATA_DIR, 'mxnet_mnist')
estimator = MXNet(entry_point=script_path,
role='SageMakerRole',
train_instance_count=1,
train_instance_type='ml.m4.xlarge',
sagemaker_session=sagemaker_session,
base_job_name='tune-mxnet')
hyperparameter_ranges = {'learning_rate': ContinuousParameter(0.01, 0.2)}
objective_metric_name = 'Validation-accuracy'
metric_definitions = [{'Name': 'Validation-accuracy', 'Regex': 'Validation-accuracy=([0-9\\.]+)'}]
tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions,
max_jobs=4, max_parallel_jobs=2)
train_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'train'),
key_prefix='integ-test-data/mxnet_mnist/train')
test_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'test'),
key_prefix='integ-test-data/mxnet_mnist/test')
tuner.fit({'train': train_input, 'test': test_input})
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
data = np.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
def test_tuning_lda(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
data_path = os.path.join(DATA_DIR, 'lda')
data_filename = 'nips-train_1.pbr'
with open(os.path.join(data_path, data_filename), 'rb') as f:
all_records = read_records(f)
# all records must be same
feature_num = int(
all_records[0].features['values'].float32_tensor.shape[0])
lda = LDA(role='SageMakerRole',
train_instance_type='ml.c4.xlarge',
num_topics=10,
sagemaker_session=sagemaker_session,
base_job_name='test-lda')
record_set = prepare_record_set_from_local_files(
data_path, lda.data_location, len(all_records), feature_num,
sagemaker_session)
test_record_set = prepare_record_set_from_local_files(
data_path, lda.data_location, len(all_records), feature_num,
sagemaker_session)
test_record_set.channel = 'test'
# specify which hp you want to optimize over
hyperparameter_ranges = {
'alpha0': ContinuousParameter(1, 10),
'num_topics': IntegerParameter(1, 2)
}
objective_metric_name = 'test:pwll'
tuner = HyperparameterTuner(
estimator=lda,
objective_metric_name=objective_metric_name,
hyperparameter_ranges=hyperparameter_ranges,
objective_type='Maximize',
max_jobs=2,
max_parallel_jobs=2)
tuner.fit([record_set, test_record_set], mini_batch_size=1)
print('Started hyperparameter tuning job with name:' +
tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job,
sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
predict_input = np.random.rand(1, feature_num)
result = predictor.predict(predict_input)
assert len(result) == 1
for record in result:
assert record.label['topic_mixture'] is not None
def test_tuning_mxnet(
sagemaker_session,
mxnet_training_latest_version,
mxnet_training_latest_py_version,
cpu_instance_type,
):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py")
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
estimator = MXNet(
entry_point=script_path,
role="SageMakerRole",
py_version=mxnet_training_latest_py_version,
instance_count=1,
instance_type=cpu_instance_type,
framework_version=mxnet_training_latest_version,
sagemaker_session=sagemaker_session,
)
hyperparameter_ranges = {
"learning-rate": ContinuousParameter(0.01, 0.2)
}
objective_metric_name = "Validation-accuracy"
metric_definitions = [{
"Name": "Validation-accuracy",
"Regex": "Validation-accuracy=([0-9\\.]+)"
}]
tuner = HyperparameterTuner(
estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
max_jobs=4,
max_parallel_jobs=2,
)
train_input = estimator.sagemaker_session.upload_data(
path=os.path.join(data_path, "train"),
key_prefix="integ-test-data/mxnet_mnist/train")
test_input = estimator.sagemaker_session.upload_data(
path=os.path.join(data_path, "test"),
key_prefix="integ-test-data/mxnet_mnist/test")
tuning_job_name = unique_name_from_base("tune-mxnet", max_length=32)
print("Started hyperparameter tuning job with name:" + tuning_job_name)
tuner.fit({
"train": train_input,
"test": test_input
},
job_name=tuning_job_name)
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job,
sagemaker_session):
predictor = tuner.deploy(1, cpu_instance_type)
data = np.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
def test_tuning_tf(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, "iris", "iris-dnn-classifier.py")
estimator = TensorFlow(
entry_point=script_path,
role="SageMakerRole",
training_steps=1,
evaluation_steps=1,
hyperparameters={"input_tensor_name": "inputs"},
train_instance_count=1,
train_instance_type="ml.c4.xlarge",
sagemaker_session=sagemaker_session,
)
inputs = sagemaker_session.upload_data(
path=DATA_PATH, key_prefix="integ-test-data/tf_iris")
hyperparameter_ranges = {
"learning_rate": ContinuousParameter(0.05, 0.2)
}
objective_metric_name = "loss"
metric_definitions = [{"Name": "loss", "Regex": "loss = ([0-9\\.]+)"}]
tuner = HyperparameterTuner(
estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
objective_type="Minimize",
max_jobs=2,
max_parallel_jobs=2,
)
tuning_job_name = unique_name_from_base("tune-tf", max_length=32)
tuner.fit(inputs, job_name=tuning_job_name)
print("Started hyperparameter tuning job with name:" + tuning_job_name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job,
sagemaker_session):
predictor = tuner.deploy(1, "ml.c4.xlarge")
features = [6.4, 3.2, 4.5, 1.5]
dict_result = predictor.predict({"inputs": features})
print("predict result: {}".format(dict_result))
list_result = predictor.predict(features)
print("predict result: {}".format(list_result))
assert dict_result == list_result
def test_tuning_kmeans(sagemaker_session):
with timeout(minutes=20):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}
# Load the data into memory as numpy arrays
with gzip.open(data_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
kmeans = KMeans(role='SageMakerRole', train_instance_count=1,
train_instance_type='ml.c4.xlarge',
k=10, sagemaker_session=sagemaker_session, base_job_name='tk',
output_path='s3://{}/'.format(sagemaker_session.default_bucket()))
# set kmeans specific hp
kmeans.init_method = 'random'
kmeans.max_iterators = 1
kmeans.tol = 1
kmeans.num_trials = 1
kmeans.local_init_method = 'kmeans++'
kmeans.half_life_time_size = 1
kmeans.epochs = 1
records = kmeans.record_set(train_set[0][:100])
test_records = kmeans.record_set(train_set[0][:100], channel='test')
# specify which hp you want to optimize over
hyperparameter_ranges = {'extra_center_factor': IntegerParameter(1, 10),
'mini_batch_size': IntegerParameter(10, 100),
'epochs': IntegerParameter(1, 2),
'init_method': CategoricalParameter(['kmeans++', 'random'])}
objective_metric_name = 'test:msd'
tuner = HyperparameterTuner(estimator=kmeans, objective_metric_name=objective_metric_name,
hyperparameter_ranges=hyperparameter_ranges, objective_type='Minimize', max_jobs=2,
max_parallel_jobs=2)
tuner.fit([records, test_records])
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
result = predictor.predict(train_set[0][:10])
assert len(result) == 10
for record in result:
assert record.label['closest_cluster'] is not None
assert record.label['distance_to_cluster'] is not None
def test_tuning_kmeans(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}
# Load the data into memory as numpy arrays
with gzip.open(data_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
kmeans = KMeans(role='SageMakerRole', train_instance_count=1,
train_instance_type='ml.c4.xlarge',
k=10, sagemaker_session=sagemaker_session, base_job_name='tk',
output_path='s3://{}/'.format(sagemaker_session.default_bucket()))
# set kmeans specific hp
kmeans.init_method = 'random'
kmeans.max_iterators = 1
kmeans.tol = 1
kmeans.num_trials = 1
kmeans.local_init_method = 'kmeans++'
kmeans.half_life_time_size = 1
kmeans.epochs = 1
records = kmeans.record_set(train_set[0][:100])
test_records = kmeans.record_set(train_set[0][:100], channel='test')
# specify which hp you want to optimize over
hyperparameter_ranges = {'extra_center_factor': IntegerParameter(1, 10),
'mini_batch_size': IntegerParameter(10, 100),
'epochs': IntegerParameter(1, 2),
'init_method': CategoricalParameter(['kmeans++', 'random'])}
objective_metric_name = 'test:msd'
tuner = HyperparameterTuner(estimator=kmeans, objective_metric_name=objective_metric_name,
hyperparameter_ranges=hyperparameter_ranges, objective_type='Minimize', max_jobs=2,
max_parallel_jobs=2)
tuner.fit([records, test_records])
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
result = predictor.predict(train_set[0][:10])
assert len(result) == 10
for record in result:
assert record.label['closest_cluster'] is not None
assert record.label['distance_to_cluster'] is not None
def test_tuning_chainer(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, 'chainer_mnist', 'mnist.py')
data_path = os.path.join(DATA_DIR, 'chainer_mnist')
estimator = Chainer(entry_point=script_path,
role='SageMakerRole',
py_version=PYTHON_VERSION,
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session,
hyperparameters={'epochs': 1})
train_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'train'),
key_prefix='integ-test-data/chainer_mnist/train')
test_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'test'),
key_prefix='integ-test-data/chainer_mnist/test')
hyperparameter_ranges = {'alpha': ContinuousParameter(0.001, 0.005)}
objective_metric_name = 'Validation-accuracy'
metric_definitions = [
{'Name': 'Validation-accuracy',
'Regex': r'\[J1\s+\d\.\d+\s+\d\.\d+\s+\d\.\d+\s+(\d\.\d+)'}]
tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges,
metric_definitions,
max_jobs=2, max_parallel_jobs=2)
tuning_job_name = unique_name_from_base('chainer', max_length=32)
tuner.fit({'train': train_input, 'test': test_input}, job_name=tuning_job_name)
print('Started hyperparameter tuning job with name:' + tuning_job_name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
batch_size = 100
data = np.zeros((batch_size, 784), dtype='float32')
output = predictor.predict(data)
assert len(output) == batch_size
data = np.zeros((batch_size, 1, 28, 28), dtype='float32')
output = predictor.predict(data)
assert len(output) == batch_size
data = np.zeros((batch_size, 28, 28), dtype='float32')
output = predictor.predict(data)
assert len(output) == batch_size
def test_tuning_tf(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, 'iris', 'iris-dnn-classifier.py')
estimator = TensorFlow(entry_point=script_path,
role='SageMakerRole',
training_steps=1,
evaluation_steps=1,
hyperparameters={'input_tensor_name': 'inputs'},
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session,
base_job_name='tune-tf')
inputs = sagemaker_session.upload_data(
path=DATA_PATH, key_prefix='integ-test-data/tf_iris')
hyperparameter_ranges = {
'learning_rate': ContinuousParameter(0.05, 0.2)
}
objective_metric_name = 'loss'
metric_definitions = [{'Name': 'loss', 'Regex': 'loss = ([0-9\\.]+)'}]
tuner = HyperparameterTuner(estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
objective_type='Minimize',
max_jobs=2,
max_parallel_jobs=2)
tuner.fit(inputs)
print('Started hyperparameter tuning job with name:' +
tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job,
sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
features = [6.4, 3.2, 4.5, 1.5]
dict_result = predictor.predict({'inputs': features})
print('predict result: {}'.format(dict_result))
list_result = predictor.predict(features)
print('predict result: {}'.format(list_result))
assert dict_result == list_result
def test_tuning_chainer(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, 'chainer_mnist', 'mnist.py')
data_path = os.path.join(DATA_DIR, 'chainer_mnist')
estimator = Chainer(entry_point=script_path,
role='SageMakerRole',
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session,
hyperparameters={'epochs': 1})
train_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'train'),
key_prefix='integ-test-data/chainer_mnist/train')
test_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'test'),
key_prefix='integ-test-data/chainer_mnist/test')
hyperparameter_ranges = {'alpha': ContinuousParameter(0.001, 0.005)}
objective_metric_name = 'Validation-accuracy'
metric_definitions = [
{'Name': 'Validation-accuracy', 'Regex': '\[J1\s+\d\.\d+\s+\d\.\d+\s+\d\.\d+\s+(\d\.\d+)'}]
tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions,
max_jobs=2, max_parallel_jobs=2)
tuner.fit({'train': train_input, 'test': test_input})
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
batch_size = 100
data = np.zeros((batch_size, 784), dtype='float32')
output = predictor.predict(data)
assert len(output) == batch_size
data = np.zeros((batch_size, 1, 28, 28), dtype='float32')
output = predictor.predict(data)
assert len(output) == batch_size
data = np.zeros((batch_size, 28, 28), dtype='float32')
output = predictor.predict(data)
assert len(output) == batch_size
def test_tuning_lda(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
data_path = os.path.join(DATA_DIR, 'lda')
data_filename = 'nips-train_1.pbr'
with open(os.path.join(data_path, data_filename), 'rb') as f:
all_records = read_records(f)
# all records must be same
feature_num = int(all_records[0].features['values'].float32_tensor.shape[0])
lda = LDA(role='SageMakerRole', train_instance_type='ml.c4.xlarge', num_topics=10,
sagemaker_session=sagemaker_session, base_job_name='test-lda')
record_set = prepare_record_set_from_local_files(data_path, lda.data_location,
len(all_records), feature_num, sagemaker_session)
test_record_set = prepare_record_set_from_local_files(data_path, lda.data_location,
len(all_records), feature_num, sagemaker_session)
test_record_set.channel = 'test'
# specify which hp you want to optimize over
hyperparameter_ranges = {'alpha0': ContinuousParameter(1, 10),
'num_topics': IntegerParameter(1, 2)}
objective_metric_name = 'test:pwll'
tuner = HyperparameterTuner(estimator=lda, objective_metric_name=objective_metric_name,
hyperparameter_ranges=hyperparameter_ranges, objective_type='Maximize', max_jobs=2,
max_parallel_jobs=2)
tuner.fit([record_set, test_record_set], mini_batch_size=1)
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
predict_input = np.random.rand(1, feature_num)
result = predictor.predict(predict_input)
assert len(result) == 1
for record in result:
assert record.label['topic_mixture'] is not None
def test_tuning_tf(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, 'iris', 'iris-dnn-classifier.py')
estimator = TensorFlow(entry_point=script_path,
role='SageMakerRole',
training_steps=1,
evaluation_steps=1,
hyperparameters={'input_tensor_name': 'inputs'},
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session,
base_job_name='tune-tf')
inputs = sagemaker_session.upload_data(path=DATA_PATH, key_prefix='integ-test-data/tf_iris')
hyperparameter_ranges = {'learning_rate': ContinuousParameter(0.05, 0.2)}
objective_metric_name = 'loss'
metric_definitions = [{'Name': 'loss', 'Regex': 'loss = ([0-9\\.]+)'}]
tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions,
objective_type='Minimize', max_jobs=2, max_parallel_jobs=2)
tuner.fit(inputs)
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
features = [6.4, 3.2, 4.5, 1.5]
dict_result = predictor.predict({'inputs': features})
print('predict result: {}'.format(dict_result))
list_result = predictor.predict(features)
print('predict result: {}'.format(list_result))
assert dict_result == list_result
def test_tuning_byo_estimator(sagemaker_session, cpu_instance_type):
"""Use Factorization Machines algorithm as an example here.
First we need to prepare data for training. We take standard data set, convert it to the
format that the algorithm can process and upload it to S3.
Then we create the Estimator and set hyperparamets as required by the algorithm.
Next, we can call fit() with path to the S3.
Later the trained model is deployed and prediction is called against the endpoint.
Default predictor is updated with json serializer and deserializer.
"""
image_uri = image_uris.retrieve("factorization-machines",
sagemaker_session.boto_region_name)
training_data_path = os.path.join(DATA_DIR, "dummy_tensor")
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
prefix = "test_byo_estimator"
key = "recordio-pb-data"
s3_train_data = sagemaker_session.upload_data(path=training_data_path,
key_prefix=os.path.join(
prefix, "train",
key))
estimator = Estimator(
image_uri=image_uri,
role="SageMakerRole",
instance_count=1,
instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
)
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type="binary_classifier")
hyperparameter_ranges = {"mini_batch_size": IntegerParameter(100, 200)}
tuner = HyperparameterTuner(
estimator=estimator,
objective_metric_name="test:binary_classification_accuracy",
hyperparameter_ranges=hyperparameter_ranges,
max_jobs=2,
max_parallel_jobs=2,
)
tuning_job_name = unique_name_from_base("byo", 32)
print("Started hyperparameter tuning job with name {}:".format(
tuning_job_name))
tuner.fit(
{
"train": s3_train_data,
"test": s3_train_data
},
include_cls_metadata=False,
job_name=tuning_job_name,
)
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job,
sagemaker_session):
predictor = tuner.deploy(
1,
cpu_instance_type,
endpoint_name=best_training_job,
serializer=_FactorizationMachineSerializer(),
deserializer=JSONDeserializer(),
)
result = predictor.predict(datasets.one_p_mnist()[0][:10])
assert len(result["predictions"]) == 10
for prediction in result["predictions"]:
assert prediction["score"] is not None
def test_tuning_chainer(sagemaker_session, chainer_latest_version,
chainer_latest_py_version, cpu_instance_type):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, "chainer_mnist", "mnist.py")
data_path = os.path.join(DATA_DIR, "chainer_mnist")
estimator = Chainer(
entry_point=script_path,
role="SageMakerRole",
framework_version=chainer_latest_version,
py_version=chainer_latest_py_version,
instance_count=1,
instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
hyperparameters={"epochs": 1},
)
train_input = estimator.sagemaker_session.upload_data(
path=os.path.join(data_path, "train"),
key_prefix="integ-test-data/chainer_mnist/train")
test_input = estimator.sagemaker_session.upload_data(
path=os.path.join(data_path, "test"),
key_prefix="integ-test-data/chainer_mnist/test")
hyperparameter_ranges = {"alpha": ContinuousParameter(0.001, 0.005)}
objective_metric_name = "Validation-accuracy"
metric_definitions = [{
"Name":
"Validation-accuracy",
"Regex":
r"\[J1\s+\d\.\d+\s+\d\.\d+\s+\d\.\d+\s+(\d\.\d+)",
}]
tuner = HyperparameterTuner(
estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
max_jobs=2,
max_parallel_jobs=2,
)
tuning_job_name = unique_name_from_base("chainer", max_length=32)
print("Started hyperparameter tuning job with name: {}".format(
tuning_job_name))
tuner.fit({
"train": train_input,
"test": test_input
},
job_name=tuning_job_name)
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job,
sagemaker_session):
predictor = tuner.deploy(1, cpu_instance_type)
batch_size = 100
data = np.zeros((batch_size, 784), dtype="float32")
output = predictor.predict(data)
assert len(output) == batch_size
data = np.zeros((batch_size, 1, 28, 28), dtype="float32")
output = predictor.predict(data)
assert len(output) == batch_size
data = np.zeros((batch_size, 28, 28), dtype="float32")
output = predictor.predict(data)
assert len(output) == batch_size
tuner = HyperparameterTuner(estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
max_jobs=9,
max_parallel_jobs=3,
objective_type=objective_type)
#Launching the tuning job
tuner.fit({'training': inputs})
#Creating endpoint
predictor = tuner.deploy(initial_instance_count=1, instance_type='ml.m4.xlarge')
#Evaluate
from IPython.display import HTML
HTML(open("input.html").read())
import numpy as np
image = np.array([data], dtype=np.float32)
response = predictor.predict(image)
prediction = response.argmax(axis=1)[0]
print(prediction)
#Cleanup
tuner.delete_endpoint()
def test_tuning_lda(sagemaker_session, cpu_instance_type):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
data_path = os.path.join(DATA_DIR, "lda")
data_filename = "nips-train_1.pbr"
with open(os.path.join(data_path, data_filename), "rb") as f:
all_records = read_records(f)
# all records must be same
feature_num = int(
all_records[0].features["values"].float32_tensor.shape[0])
lda = LDA(
role="SageMakerRole",
instance_type=cpu_instance_type,
num_topics=10,
sagemaker_session=sagemaker_session,
)
record_set = prepare_record_set_from_local_files(
data_path, lda.data_location, len(all_records), feature_num,
sagemaker_session)
test_record_set = prepare_record_set_from_local_files(
data_path, lda.data_location, len(all_records), feature_num,
sagemaker_session)
test_record_set.channel = "test"
# specify which hp you want to optimize over
hyperparameter_ranges = {
"alpha0": ContinuousParameter(1, 10),
"num_topics": IntegerParameter(1, 2),
}
objective_metric_name = "test:pwll"
tuner = HyperparameterTuner(
estimator=lda,
objective_metric_name=objective_metric_name,
hyperparameter_ranges=hyperparameter_ranges,
objective_type="Maximize",
max_jobs=2,
max_parallel_jobs=2,
early_stopping_type="Auto",
)
tuning_job_name = unique_name_from_base("test-lda", max_length=32)
print("Started hyperparameter tuning job with name:" + tuning_job_name)
tuner.fit([record_set, test_record_set],
mini_batch_size=1,
job_name=tuning_job_name)
attached_tuner = HyperparameterTuner.attach(
tuning_job_name, sagemaker_session=sagemaker_session)
assert attached_tuner.early_stopping_type == "Auto"
assert attached_tuner.estimator.alpha0 == 1.0
assert attached_tuner.estimator.num_topics == 1
best_training_job = attached_tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job,
sagemaker_session):
predictor = tuner.deploy(1, cpu_instance_type)
predict_input = np.random.rand(1, feature_num)
result = predictor.predict(predict_input)
assert len(result) == 1
for record in result:
assert record.label["topic_mixture"] is not None
def test_tuning_byo_estimator(sagemaker_session):
"""Use Factorization Machines algorithm as an example here.
First we need to prepare data for training. We take standard data set, convert it to the
format that the algorithm can process and upload it to S3.
Then we create the Estimator and set hyperparamets as required by the algorithm.
Next, we can call fit() with path to the S3.
Later the trained model is deployed and prediction is called against the endpoint.
Default predictor is updated with json serializer and deserializer.
"""
image_name = registry(sagemaker_session.boto_session.region_name
) + "/factorization-machines:1"
training_data_path = os.path.join(DATA_DIR, "dummy_tensor")
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
data_path = os.path.join(DATA_DIR, "one_p_mnist", "mnist.pkl.gz")
pickle_args = {} if sys.version_info.major == 2 else {
"encoding": "latin1"
}
with gzip.open(data_path, "rb") as f:
train_set, _, _ = pickle.load(f, **pickle_args)
prefix = "test_byo_estimator"
key = "recordio-pb-data"
s3_train_data = sagemaker_session.upload_data(path=training_data_path,
key_prefix=os.path.join(
prefix, "train",
key))
estimator = Estimator(
image_name=image_name,
role="SageMakerRole",
train_instance_count=1,
train_instance_type="ml.c4.xlarge",
sagemaker_session=sagemaker_session,
)
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type="binary_classifier")
hyperparameter_ranges = {"mini_batch_size": IntegerParameter(100, 200)}
tuner = HyperparameterTuner(
estimator=estimator,
objective_metric_name="test:binary_classification_accuracy",
hyperparameter_ranges=hyperparameter_ranges,
max_jobs=2,
max_parallel_jobs=2,
)
tuner.fit(
{
"train": s3_train_data,
"test": s3_train_data
},
include_cls_metadata=False,
job_name=unique_name_from_base("byo", 32),
)
print("Started hyperparameter tuning job with name:" +
tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job,
sagemaker_session):
predictor = tuner.deploy(1,
"ml.m4.xlarge",
endpoint_name=best_training_job)
predictor.serializer = _fm_serializer
predictor.content_type = "application/json"
predictor.deserializer = json_deserializer
result = predictor.predict(train_set[0][:10])
assert len(result["predictions"]) == 10
for prediction in result["predictions"]:
assert prediction["score"] is not None
def test_tuning_byo_estimator(sagemaker_session):
"""Use Factorization Machines algorithm as an example here.
First we need to prepare data for training. We take standard data set, convert it to the
format that the algorithm can process and upload it to S3.
Then we create the Estimator and set hyperparamets as required by the algorithm.
Next, we can call fit() with path to the S3.
Later the trained model is deployed and prediction is called against the endpoint.
Default predictor is updated with json serializer and deserializer.
"""
image_name = registry(sagemaker_session.boto_session.region_name
) + '/factorization-machines:1'
training_data_path = os.path.join(DATA_DIR, 'dummy_tensor')
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
pickle_args = {} if sys.version_info.major == 2 else {
'encoding': 'latin1'
}
with gzip.open(data_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
prefix = 'test_byo_estimator'
key = 'recordio-pb-data'
s3_train_data = sagemaker_session.upload_data(path=training_data_path,
key_prefix=os.path.join(
prefix, 'train',
key))
estimator = Estimator(image_name=image_name,
role='SageMakerRole',
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session,
base_job_name='test-byo')
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type='binary_classifier')
hyperparameter_ranges = {'mini_batch_size': IntegerParameter(100, 200)}
tuner = HyperparameterTuner(
estimator=estimator,
base_tuning_job_name='byo',
objective_metric_name='test:binary_classification_accuracy',
hyperparameter_ranges=hyperparameter_ranges,
max_jobs=2,
max_parallel_jobs=2)
tuner.fit({
'train': s3_train_data,
'test': s3_train_data
},
include_cls_metadata=False)
print('Started hyperparameter tuning job with name:' +
tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job,
sagemaker_session):
predictor = tuner.deploy(1,
'ml.m4.xlarge',
endpoint_name=best_training_job)
predictor.serializer = _fm_serializer
predictor.content_type = 'application/json'
predictor.deserializer = json_deserializer
result = predictor.predict(train_set[0][:10])
assert len(result['predictions']) == 10
for prediction in result['predictions']:
assert prediction['score'] is not None
def test_tuning_byo_estimator(sagemaker_session):
"""Use Factorization Machines algorithm as an example here.
First we need to prepare data for training. We take standard data set, convert it to the
format that the algorithm can process and upload it to S3.
Then we create the Estimator and set hyperparamets as required by the algorithm.
Next, we can call fit() with path to the S3.
Later the trained model is deployed and prediction is called against the endpoint.
Default predictor is updated with json serializer and deserializer.
"""
image_name = registry(sagemaker_session.boto_session.region_name) + '/factorization-machines:1'
training_data_path = os.path.join(DATA_DIR, 'dummy_tensor')
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}
with gzip.open(data_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
prefix = 'test_byo_estimator'
key = 'recordio-pb-data'
s3_train_data = sagemaker_session.upload_data(path=training_data_path,
key_prefix=os.path.join(prefix, 'train', key))
estimator = Estimator(image_name=image_name,
role='SageMakerRole', train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session, base_job_name='test-byo')
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type='binary_classifier')
hyperparameter_ranges = {'mini_batch_size': IntegerParameter(100, 200)}
tuner = HyperparameterTuner(estimator=estimator, base_tuning_job_name='byo',
objective_metric_name='test:binary_classification_accuracy',
hyperparameter_ranges=hyperparameter_ranges,
max_jobs=2, max_parallel_jobs=2)
tuner.fit({'train': s3_train_data, 'test': s3_train_data}, include_cls_metadata=False)
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.m4.xlarge', endpoint_name=best_training_job)
predictor.serializer = _fm_serializer
predictor.content_type = 'application/json'
predictor.deserializer = json_deserializer
result = predictor.predict(train_set[0][:10])
assert len(result['predictions']) == 10
for prediction in result['predictions']:
assert prediction['score'] is not None
p = figure(plot_width=600, plot_height=600,
title="Objective vs %s" % hp_name,
tools=hover.tools(),
x_axis_label=hp_name, y_axis_label=objective_name,
**categorical_args)
p.circle(source=df, x=hp_name, y='FinalObjectiveValue')
figures.append(p)
show(bokeh.layouts.Column(*figures))
# Deploy this as your final model and evaluate it on the test set.
# In[57]:
tuned_model_deploy = tuner.deploy(initial_instance_count=1, instance_type='ml.m4.xlarge')
# In[58]:
predict_batches(tuned_model_deploy, test_features, test_labels)
# In[59]:
predict_batches(tuned_model_deploy, val_features, val_labels)
# ### OPTIONAL: Try the XGBoost algorithm
