def _tune(
kmeans_estimator,
kmeans_train_set,
tuner=None,
hyperparameter_ranges=None,
job_name=None,
warm_start_config=None,
wait=True,
max_jobs=2,
max_parallel_jobs=2,
early_stopping_type="Off",
):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
if not tuner:
tuner = HyperparameterTuner(
estimator=kmeans_estimator,
objective_metric_name="test:msd",
hyperparameter_ranges=hyperparameter_ranges,
objective_type="Minimize",
max_jobs=max_jobs,
max_parallel_jobs=max_parallel_jobs,
warm_start_config=warm_start_config,
early_stopping_type=early_stopping_type,
)
records = kmeans_estimator.record_set(kmeans_train_set[0][:100])
test_record_set = kmeans_estimator.record_set(
kmeans_train_set[0][:100], channel="test")
print(
"Started hyperparameter tuning job with name: {}".format(job_name))
tuner.fit([records, test_record_set], job_name=job_name, wait=wait)
return tuner
def test_stop_tuning_job(sagemaker_session):
feature_num = 14
train_input = np.random.rand(1000, feature_num)
rcf = RandomCutForest(role='SageMakerRole', train_instance_count=1, train_instance_type='ml.c4.xlarge',
num_trees=50, num_samples_per_tree=20, sagemaker_session=sagemaker_session,
base_job_name='test-randomcutforest')
records = rcf.record_set(train_input)
records.distribution = 'FullyReplicated'
test_records = rcf.record_set(train_input, channel='test')
test_records.distribution = 'FullyReplicated'
hyperparameter_ranges = {'num_trees': IntegerParameter(50, 100),
'num_samples_per_tree': IntegerParameter(1, 2)}
objective_metric_name = 'test:f1'
tuner = HyperparameterTuner(estimator=rcf, objective_metric_name=objective_metric_name,
hyperparameter_ranges=hyperparameter_ranges, objective_type='Maximize', max_jobs=2,
max_parallel_jobs=2)
tuner.fit([records, test_records])
time.sleep(15)
latest_tuning_job_name = tuner.latest_tuning_job.name
print('Attempting to stop {}'.format(latest_tuning_job_name))
tuner.stop_tuning_job()
desc = tuner.latest_tuning_job.sagemaker_session.sagemaker_client\
.describe_hyper_parameter_tuning_job(HyperParameterTuningJobName=latest_tuning_job_name)
assert desc['HyperParameterTuningJobStatus'] == 'Stopping'
def model_fit(
self,
inputs: Dict[str, str],
hparam: Dict[str, Any] = None,
) -> None:
if hparam is not None:
tuner = HyperparameterTuner(
estimator=self.estimator,
objective_metric_name=hparam.get('objective_metric_name'),
metric_definitions=hparam.get('metric_definitions'),
hyperparameter_ranges=hparam.get('hyperparameter_ranges'),
objective_type=hparam.get('objective_type'),
max_jobs=hparam.get('max_jobs'),
max_parallel_jobs=hparam.get('max_parallel_jobs'),
tags=self._project_tag,
base_tuning_job_name=self._training_job_name,
)
tuner.fit(
inputs=inputs,
job_name=self._training_job_name,
wait=False,
logs='All',
)
else:
self.estimator.fit(
inputs=inputs,
job_name=self._training_job_name,
wait=False,
logs='All',
)
def test_tuning_tf_vpc_multi(
sagemaker_session,
cpu_instance_type,
tensorflow_training_latest_version,
tensorflow_training_latest_py_version,
):
"""Test Tensorflow multi-instance using the same VpcConfig for training and inference"""
instance_type = cpu_instance_type
instance_count = 2
resource_path = os.path.join(DATA_DIR, "tensorflow_mnist")
script_path = "mnist.py"
ec2_client = sagemaker_session.boto_session.client("ec2")
subnet_ids, security_group_id = vpc_test_utils.get_or_create_vpc_resources(
ec2_client)
vpc_test_utils.setup_security_group_for_encryption(ec2_client,
security_group_id)
estimator = TensorFlow(
entry_point=script_path,
source_dir=resource_path,
role="SageMakerRole",
framework_version=tensorflow_training_latest_version,
py_version=tensorflow_training_latest_py_version,
instance_count=instance_count,
instance_type=instance_type,
sagemaker_session=sagemaker_session,
base_job_name="test-vpc-tf",
subnets=subnet_ids,
security_group_ids=[security_group_id],
encrypt_inter_container_traffic=True,
)
hyperparameter_ranges = {"epochs": IntegerParameter(1, 2)}
objective_metric_name = "accuracy"
metric_definitions = [{
"Name": objective_metric_name,
"Regex": "accuracy = ([0-9\\.]+)"
}]
tuner = HyperparameterTuner(
estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
max_jobs=2,
max_parallel_jobs=2,
)
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
inputs = estimator.sagemaker_session.upload_data(
path=os.path.join(resource_path, "data"),
key_prefix="scriptmode/mnist")
tuning_job_name = unique_name_from_base("tune-tf", max_length=32)
print(
f"Started hyperparameter tuning job with name: {tuning_job_name}")
tuner.fit(inputs, job_name=tuning_job_name)
def test_tuning_step_with_single_algo_tuner(pipeline_session, entry_point):
inputs = TrainingInput(
s3_data=f"s3://{pipeline_session.default_bucket()}/training-data")
pytorch_estimator = PyTorch(
entry_point=entry_point,
role=sagemaker.get_execution_role(),
framework_version="1.5.0",
py_version="py3",
instance_count=1,
instance_type="ml.m5.xlarge",
sagemaker_session=pipeline_session,
enable_sagemaker_metrics=True,
max_retry_attempts=3,
)
hyperparameter_ranges = {
"batch-size": IntegerParameter(64, 128),
}
tuner = HyperparameterTuner(
estimator=pytorch_estimator,
objective_metric_name="test:acc",
objective_type="Maximize",
hyperparameter_ranges=hyperparameter_ranges,
metric_definitions=[{
"Name": "test:acc",
"Regex": "Overall test accuracy: (.*?);"
}],
max_jobs=2,
max_parallel_jobs=2,
)
with warnings.catch_warnings(record=True) as w:
step_args = tuner.fit(inputs=inputs)
assert len(w) == 1
assert issubclass(w[-1].category, UserWarning)
assert "Running within a PipelineSession" in str(w[-1].message)
with warnings.catch_warnings(record=True) as w:
step = TuningStep(
name="MyTuningStep",
step_args=step_args,
)
assert len(w) == 0
pipeline = Pipeline(
name="MyPipeline",
steps=[step],
sagemaker_session=pipeline_session,
)
assert json.loads(pipeline.definition())["Steps"][0] == {
"Name": "MyTuningStep",
"Type": "Tuning",
"Arguments": step_args,
}
def test_tuning_mxnet(sagemaker_session):
with timeout(minutes=15):
script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'tuning.py')
data_path = os.path.join(DATA_DIR, 'mxnet_mnist')
estimator = MXNet(entry_point=script_path,
role='SageMakerRole',
train_instance_count=1,
train_instance_type='ml.m4.xlarge',
sagemaker_session=sagemaker_session,
base_job_name='tune-mxnet')
hyperparameter_ranges = {'learning_rate': ContinuousParameter(0.01, 0.2)}
objective_metric_name = 'Validation-accuracy'
metric_definitions = [{'Name': 'Validation-accuracy', 'Regex': 'Validation-accuracy=([0-9\\.]+)'}]
tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions,
max_jobs=4, max_parallel_jobs=2)
train_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'train'),
key_prefix='integ-test-data/mxnet_mnist/train')
test_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'test'),
key_prefix='integ-test-data/mxnet_mnist/test')
tuner.fit({'train': train_input, 'test': test_input})
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
data = np.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
def sagemaker_hyperparam_tuning(sm_estimator, train_s3, hyperparameter_ranges,
metric_definitions, tuning_job_name, max_jobs,
max_parallel_jobs):
objective_metric_name = 'validation:error'
objective_type = 'Minimize'
tuner = HyperparameterTuner(estimator=sm_estimator,
objective_metric_name=objective_metric_name,
hyperparameter_ranges=hyperparameter_ranges,
metric_definitions=metric_definitions,
max_jobs=max_jobs,
max_parallel_jobs=max_parallel_jobs,
objective_type=objective_type)
tuner.fit(train_s3, job_name=tuning_job_name, wait=False)
def test_attach_tuning_pytorch(sagemaker_session):
mnist_dir = os.path.join(DATA_DIR, "pytorch_mnist")
mnist_script = os.path.join(mnist_dir, "mnist.py")
estimator = PyTorch(
entry_point=mnist_script,
role="SageMakerRole",
train_instance_count=1,
py_version=PYTHON_VERSION,
train_instance_type="ml.c4.xlarge",
sagemaker_session=sagemaker_session,
)
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
objective_metric_name = "evaluation-accuracy"
metric_definitions = [{
"Name": "evaluation-accuracy",
"Regex": r"Overall test accuracy: (\d+)"
}]
hyperparameter_ranges = {"batch-size": IntegerParameter(50, 100)}
tuner = HyperparameterTuner(
estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
max_jobs=2,
max_parallel_jobs=2,
early_stopping_type="Auto",
)
training_data = estimator.sagemaker_session.upload_data(
path=os.path.join(mnist_dir, "training"),
key_prefix="integ-test-data/pytorch_mnist/training",
)
tuning_job_name = unique_name_from_base("pytorch", max_length=32)
tuner.fit({"training": training_data}, job_name=tuning_job_name)
print("Started hyperparameter tuning job with name:" + tuning_job_name)
time.sleep(15)
tuner.wait()
attached_tuner = HyperparameterTuner.attach(
tuning_job_name, sagemaker_session=sagemaker_session)
assert attached_tuner.early_stopping_type == "Auto"
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job,
sagemaker_session):
predictor = attached_tuner.deploy(1, "ml.c4.xlarge")
data = np.zeros(shape=(1, 1, 28, 28), dtype=np.float32)
predictor.predict(data)
batch_size = 100
data = np.random.rand(batch_size, 1, 28, 28).astype(np.float32)
output = predictor.predict(data)
assert output.shape == (batch_size, 10)
def tune(
self,
tuning_job_parameters: Dict[str, Any] = {},
hyperparameters: Dict[str, Any] = None,
hyperparameter_tuning: Dict[str, Any] = None,
):
"""
Tunes the current Estimator with the provided hyperparameters
"""
LOGGER.info("Starting the hyperparameter tuning.")
if hyperparameters is None:
hyperparameters = self.default_hyperparameters
hyperparameters["feature_dim"] = len(self.data.feature_columns)
if hyperparameter_tuning is None:
hyperparameter_tuning = self.default_hyperparameter_tuning
used_tuning_job_parameters = {
**self.default_tuning_job_config,
**tuning_job_parameters,
}
if self._model is None:
self._model = self._get_model(hyperparameters)
self._tuner = HyperparameterTuner(
estimator=self._model,
**used_tuning_job_parameters,
hyperparameter_ranges=hyperparameter_tuning,
)
# Get the data and upload to S3
Y_train = self.data.train_data.loc[:, self.data.output_column]
X_train = self.data.train_data.loc[:, self.data.feature_columns]
s3_input_train = self._prepare_data("train", X_train, Y_train)
Y_validation = self.data.validation_data.loc[:,
self.data.output_column]
X_validation = self.data.validation_data.loc[:,
self.data.feature_columns]
s3_input_validation = self._prepare_data("validation", X_validation,
Y_validation)
LOGGER.info("Starting to tune hyperparameters")
self._tuner.fit({
"train": s3_input_train,
"validation": s3_input_validation
})
self._tuner.wait()
LOGGER.info("Done with the tuning job")
def test_tuning_lda(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
data_path = os.path.join(DATA_DIR, 'lda')
data_filename = 'nips-train_1.pbr'
with open(os.path.join(data_path, data_filename), 'rb') as f:
all_records = read_records(f)
# all records must be same
feature_num = int(
all_records[0].features['values'].float32_tensor.shape[0])
lda = LDA(role='SageMakerRole',
train_instance_type='ml.c4.xlarge',
num_topics=10,
sagemaker_session=sagemaker_session,
base_job_name='test-lda')
record_set = prepare_record_set_from_local_files(
data_path, lda.data_location, len(all_records), feature_num,
sagemaker_session)
test_record_set = prepare_record_set_from_local_files(
data_path, lda.data_location, len(all_records), feature_num,
sagemaker_session)
test_record_set.channel = 'test'
# specify which hp you want to optimize over
hyperparameter_ranges = {
'alpha0': ContinuousParameter(1, 10),
'num_topics': IntegerParameter(1, 2)
}
objective_metric_name = 'test:pwll'
tuner = HyperparameterTuner(
estimator=lda,
objective_metric_name=objective_metric_name,
hyperparameter_ranges=hyperparameter_ranges,
objective_type='Maximize',
max_jobs=2,
max_parallel_jobs=2)
tuner.fit([record_set, test_record_set], mini_batch_size=1)
print('Started hyperparameter tuning job with name:' +
tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job,
sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
predict_input = np.random.rand(1, feature_num)
result = predictor.predict(predict_input)
assert len(result) == 1
for record in result:
assert record.label['topic_mixture'] is not None
def test_tuning_mxnet(sagemaker_session, mxnet_full_version,
cpu_instance_type):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, "mxnet_mnist", "mnist.py")
data_path = os.path.join(DATA_DIR, "mxnet_mnist")
estimator = MXNet(
entry_point=script_path,
role="SageMakerRole",
py_version=PYTHON_VERSION,
train_instance_count=1,
train_instance_type=cpu_instance_type,
framework_version=mxnet_full_version,
sagemaker_session=sagemaker_session,
)
hyperparameter_ranges = {
"learning-rate": ContinuousParameter(0.01, 0.2)
}
objective_metric_name = "Validation-accuracy"
metric_definitions = [{
"Name": "Validation-accuracy",
"Regex": "Validation-accuracy=([0-9\\.]+)"
}]
tuner = HyperparameterTuner(
estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
max_jobs=4,
max_parallel_jobs=2,
)
train_input = estimator.sagemaker_session.upload_data(
path=os.path.join(data_path, "train"),
key_prefix="integ-test-data/mxnet_mnist/train")
test_input = estimator.sagemaker_session.upload_data(
path=os.path.join(data_path, "test"),
key_prefix="integ-test-data/mxnet_mnist/test")
tuning_job_name = unique_name_from_base("tune-mxnet", max_length=32)
tuner.fit({
"train": train_input,
"test": test_input
},
job_name=tuning_job_name)
print("Started hyperparameter tuning job with name:" + tuning_job_name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job,
sagemaker_session):
predictor = tuner.deploy(1, cpu_instance_type)
data = np.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
def test_fit_1p(sagemaker_session):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}
# Load the data into memory as numpy arrays
with gzip.open(data_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
kmeans = KMeans(role='SageMakerRole',
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
k=10,
sagemaker_session=sagemaker_session,
base_job_name='tk',
output_path='s3://{}/'.format(
sagemaker_session.default_bucket()))
# set kmeans specific hp
kmeans.init_method = 'random'
kmeans.max_iterators = 1
kmeans.tol = 1
kmeans.num_trials = 1
kmeans.local_init_method = 'kmeans++'
kmeans.half_life_time_size = 1
kmeans.epochs = 1
records = kmeans.record_set(train_set[0][:100])
test_records = kmeans.record_set(train_set[0][:100], channel='test')
# specify which hp you want to optimize over
hyperparameter_ranges = {
'extra_center_factor': IntegerParameter(1, 10),
'mini_batch_size': IntegerParameter(10, 100),
'epochs': IntegerParameter(1, 2),
'init_method': CategoricalParameter(['kmeans++', 'random'])
}
objective_metric_name = 'test:msd'
tuner = HyperparameterTuner(estimator=kmeans,
objective_metric_name=objective_metric_name,
hyperparameter_ranges=hyperparameter_ranges,
objective_type='Minimize',
max_jobs=2,
max_parallel_jobs=2)
tuner.fit([records, test_records])
print('Started HPO job with name:' + tuner.latest_tuning_job.name)
def test_multi_estimator_tuning(sagemaker_session, estimator_fm, estimator_knn,
data_set, cpu_instance_type):
tuner = HyperparameterTuner.create(
base_tuning_job_name=BASE_TUNING_JOB_NAME,
estimator_dict={
ESTIMATOR_FM: estimator_fm,
ESTIMATOR_KNN: estimator_knn
},
objective_metric_name_dict={
ESTIMATOR_FM: OBJECTIVE_METRIC_NAME_FM,
ESTIMATOR_KNN: OBJECTIVE_METRIC_NAME_KNN,
},
hyperparameter_ranges_dict={
ESTIMATOR_FM: HYPER_PARAMETER_RANGES_FM,
ESTIMATOR_KNN: HYPER_PARAMETER_RANGES_KNN,
},
strategy=STRATEGY,
objective_type=OBJECTIVE_TYPE,
max_jobs=MAX_JOBS,
max_parallel_jobs=MAX_PARALLEL_JOBS,
tags=TAGS,
)
_fit_tuner(sagemaker_session, tuner)
_retrieve_analytics(sagemaker_session, tuner.latest_tuning_job.name)
tuner_attached = _attach_tuner(sagemaker_session,
tuner.latest_tuning_job.name)
_deploy_and_predict(sagemaker_session, tuner_attached, data_set,
cpu_instance_type)
def get_xgb_tuner(output_path, model_name):
xgb = _init_model(role, output_path, model_name)
# Set core hyperparameters
xgb.set_hyperparameters(
eval_metric='rmse',
objective=
'reg:linear', # plenty of options out there: https://github.com/dmlc/xgboost/blob/master/doc/parameter.rst#learning-task-parameters
num_round=100,
rate_drop=0.3,
tweedie_variance_power=1.4)
hyperparemeters_to_tune = {
'eta': ContinuousParameter(0, 1),
'min_child_weight': ContinuousParameter(1, 10),
'alpha': ContinuousParameter(0, 2),
'max_depth': IntegerParameter(1, 10)
}
tuner = HyperparameterTuner(
xgb,
'validation:rmse', # objective metric
hyperparemeters_to_tune,
max_jobs=20,
max_parallel_jobs=3,
base_tuning_job_name=model_name + "-tuner",
objective_type='Minimize')
return tuner
def test_attach_tuning_job_with_estimator_from_kwarg(sagemaker_session):
job_details = copy.deepcopy(TUNING_JOB_DETAILS)
sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job',
return_value=job_details)
tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session,
estimator_cls='sagemaker.estimator.Estimator')
assert isinstance(tuner.estimator, Estimator)
def test_attach_tuning_job_with_estimator_from_hyperparameters(sagemaker_session):
job_details = copy.deepcopy(TUNING_JOB_DETAILS)
sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(
name="describe_tuning_job", return_value=job_details
)
tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session)
assert tuner.latest_tuning_job.name == JOB_NAME
assert tuner.objective_metric_name == OBJECTIVE_METRIC_NAME
assert tuner.max_jobs == 1
assert tuner.max_parallel_jobs == 1
assert tuner.metric_definitions == METRIC_DEFINITIONS
assert tuner.strategy == "Bayesian"
assert tuner.objective_type == "Minimize"
assert tuner.early_stopping_type == "Off"
assert isinstance(tuner.estimator, PCA)
assert tuner.estimator.role == ROLE
assert tuner.estimator.train_instance_count == 1
assert tuner.estimator.train_max_run == 24 * 60 * 60
assert tuner.estimator.input_mode == "File"
assert tuner.estimator.output_path == BUCKET_NAME
assert tuner.estimator.output_kms_key == ""
assert "_tuning_objective_metric" not in tuner.estimator.hyperparameters()
assert tuner.estimator.hyperparameters()["num_components"] == "10"
def test_attach_tuning_job_with_estimator_from_kwarg(sagemaker_session):
job_details = copy.deepcopy(TUNING_JOB_DETAILS)
sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job',
return_value=job_details)
tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session,
estimator_cls='sagemaker.estimator.Estimator')
assert isinstance(tuner.estimator, Estimator)
def test_attach_tuning_job_with_estimator_from_image(sagemaker_session):
job_details = copy.deepcopy(TUNING_JOB_DETAILS)
job_details['TrainingJobDefinition']['AlgorithmSpecification']['TrainingImage'] = '1111.amazonaws.com/pca:1'
sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job',
return_value=job_details)
tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session)
assert isinstance(tuner.estimator, PCA)
def test_attach_tuning_job_with_estimator_from_image(sagemaker_session):
job_details = copy.deepcopy(TUNING_JOB_DETAILS)
job_details['TrainingJobDefinition']['AlgorithmSpecification']['TrainingImage'] = '1111.amazonaws.com/pca:1'
sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job',
return_value=job_details)
tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session)
assert isinstance(tuner.estimator, PCA)
def test_tuning_tf(
sagemaker_session,
cpu_instance_type,
tensorflow_training_latest_version,
tensorflow_training_latest_py_version,
):
resource_path = os.path.join(DATA_DIR, "tensorflow_mnist")
script_path = "mnist.py"
estimator = TensorFlow(
entry_point=script_path,
source_dir=resource_path,
role="SageMakerRole",
instance_count=1,
instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
framework_version=tensorflow_training_latest_version,
py_version=tensorflow_training_latest_py_version,
)
hyperparameter_ranges = {"epochs": IntegerParameter(1, 2)}
objective_metric_name = "accuracy"
metric_definitions = [{
"Name": objective_metric_name,
"Regex": "accuracy = ([0-9\\.]+)"
}]
tuner = HyperparameterTuner(
estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
max_jobs=2,
max_parallel_jobs=2,
)
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
inputs = estimator.sagemaker_session.upload_data(
path=os.path.join(resource_path, "data"),
key_prefix="scriptmode/mnist")
tuning_job_name = unique_name_from_base("tune-tf", max_length=32)
print("Started hyperparameter tuning job with name: " +
tuning_job_name)
tuner.fit(inputs, job_name=tuning_job_name)
def test_attach_tuning_pytorch(sagemaker_session):
mnist_dir = os.path.join(DATA_DIR, 'pytorch_mnist')
mnist_script = os.path.join(mnist_dir, 'mnist.py')
estimator = PyTorch(entry_point=mnist_script,
role='SageMakerRole',
train_instance_count=1,
py_version=PYTHON_VERSION,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session)
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
objective_metric_name = 'evaluation-accuracy'
metric_definitions = [{
'Name': 'evaluation-accuracy',
'Regex': r'Overall test accuracy: (\d+)'
}]
hyperparameter_ranges = {'batch-size': IntegerParameter(50, 100)}
tuner = HyperparameterTuner(estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
max_jobs=2,
max_parallel_jobs=2,
early_stopping_type='Auto')
training_data = estimator.sagemaker_session.upload_data(
path=os.path.join(mnist_dir, 'training'),
key_prefix='integ-test-data/pytorch_mnist/training')
tuning_job_name = unique_name_from_base('pytorch', max_length=32)
tuner.fit({'training': training_data}, job_name=tuning_job_name)
print('Started hyperparameter tuning job with name:' + tuning_job_name)
time.sleep(15)
tuner.wait()
attached_tuner = HyperparameterTuner.attach(
tuning_job_name, sagemaker_session=sagemaker_session)
assert attached_tuner.early_stopping_type == 'Auto'
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job,
sagemaker_session):
predictor = attached_tuner.deploy(1, 'ml.c4.xlarge')
data = np.zeros(shape=(1, 1, 28, 28), dtype=np.float32)
predictor.predict(data)
batch_size = 100
data = np.random.rand(batch_size, 1, 28, 28).astype(np.float32)
output = predictor.predict(data)
assert output.shape == (batch_size, 10)
def test_tuning_mxnet(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, 'mxnet_mnist', 'tuning.py')
data_path = os.path.join(DATA_DIR, 'mxnet_mnist')
estimator = MXNet(entry_point=script_path,
role='SageMakerRole',
train_instance_count=1,
train_instance_type='ml.m4.xlarge',
sagemaker_session=sagemaker_session,
base_job_name='tune-mxnet')
hyperparameter_ranges = {'learning_rate': ContinuousParameter(0.01, 0.2)}
objective_metric_name = 'Validation-accuracy'
metric_definitions = [{'Name': 'Validation-accuracy', 'Regex': 'Validation-accuracy=([0-9\\.]+)'}]
tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions,
max_jobs=4, max_parallel_jobs=2)
train_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'train'),
key_prefix='integ-test-data/mxnet_mnist/train')
test_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'test'),
key_prefix='integ-test-data/mxnet_mnist/test')
tuner.fit({'train': train_input, 'test': test_input})
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
data = np.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
def test_attach_with_no_specified_estimator(sagemaker_session):
job_details = copy.deepcopy(TUNING_JOB_DETAILS)
del job_details['TrainingJobDefinition']['StaticHyperParameters']['sagemaker_estimator_module']
del job_details['TrainingJobDefinition']['StaticHyperParameters']['sagemaker_estimator_class_name']
sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job',
return_value=job_details)
tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session)
assert isinstance(tuner.estimator, Estimator)
def test_identical_dataset_and_algorithm_tuner(sagemaker_session):
job_details = copy.deepcopy(TUNING_JOB_DETAILS)
sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job',
return_value=job_details)
tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session)
parent_tuner = tuner.identical_dataset_and_algorithm_tuner(additional_parents={"p1", "p2"})
assert parent_tuner.warm_start_config.type == WarmStartTypes.IDENTICAL_DATA_AND_ALGORITHM
assert parent_tuner.warm_start_config.parents == {tuner.latest_tuning_job.name, "p1", "p2"}
def test_attach_with_no_specified_estimator(sagemaker_session):
job_details = copy.deepcopy(TUNING_JOB_DETAILS)
del job_details['TrainingJobDefinition']['StaticHyperParameters']['sagemaker_estimator_module']
del job_details['TrainingJobDefinition']['StaticHyperParameters']['sagemaker_estimator_class_name']
sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job',
return_value=job_details)
tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session)
assert isinstance(tuner.estimator, Estimator)
def test_tuning_tf(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, "iris", "iris-dnn-classifier.py")
estimator = TensorFlow(
entry_point=script_path,
role="SageMakerRole",
training_steps=1,
evaluation_steps=1,
hyperparameters={"input_tensor_name": "inputs"},
train_instance_count=1,
train_instance_type="ml.c4.xlarge",
sagemaker_session=sagemaker_session,
)
inputs = sagemaker_session.upload_data(
path=DATA_PATH, key_prefix="integ-test-data/tf_iris")
hyperparameter_ranges = {
"learning_rate": ContinuousParameter(0.05, 0.2)
}
objective_metric_name = "loss"
metric_definitions = [{"Name": "loss", "Regex": "loss = ([0-9\\.]+)"}]
tuner = HyperparameterTuner(
estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
objective_type="Minimize",
max_jobs=2,
max_parallel_jobs=2,
)
tuning_job_name = unique_name_from_base("tune-tf", max_length=32)
tuner.fit(inputs, job_name=tuning_job_name)
print("Started hyperparameter tuning job with name:" + tuning_job_name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job,
sagemaker_session):
predictor = tuner.deploy(1, "ml.c4.xlarge")
features = [6.4, 3.2, 4.5, 1.5]
dict_result = predictor.predict({"inputs": features})
print("predict result: {}".format(dict_result))
list_result = predictor.predict(features)
print("predict result: {}".format(list_result))
assert dict_result == list_result
def test_tuning_chainer(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, 'chainer_mnist', 'mnist.py')
data_path = os.path.join(DATA_DIR, 'chainer_mnist')
estimator = Chainer(entry_point=script_path,
role='SageMakerRole',
py_version=PYTHON_VERSION,
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session,
hyperparameters={'epochs': 1})
train_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'train'),
key_prefix='integ-test-data/chainer_mnist/train')
test_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'test'),
key_prefix='integ-test-data/chainer_mnist/test')
hyperparameter_ranges = {'alpha': ContinuousParameter(0.001, 0.005)}
objective_metric_name = 'Validation-accuracy'
metric_definitions = [
{'Name': 'Validation-accuracy',
'Regex': r'\[J1\s+\d\.\d+\s+\d\.\d+\s+\d\.\d+\s+(\d\.\d+)'}]
tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges,
metric_definitions,
max_jobs=2, max_parallel_jobs=2)
tuning_job_name = unique_name_from_base('chainer', max_length=32)
tuner.fit({'train': train_input, 'test': test_input}, job_name=tuning_job_name)
print('Started hyperparameter tuning job with name:' + tuning_job_name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
batch_size = 100
data = np.zeros((batch_size, 784), dtype='float32')
output = predictor.predict(data)
assert len(output) == batch_size
data = np.zeros((batch_size, 1, 28, 28), dtype='float32')
output = predictor.predict(data)
assert len(output) == batch_size
data = np.zeros((batch_size, 28, 28), dtype='float32')
output = predictor.predict(data)
assert len(output) == batch_size
def test_validate_parameter_ranges_number_validation_error(sagemaker_session):
pca = PCA(ROLE, TRAIN_INSTANCE_COUNT, TRAIN_INSTANCE_TYPE, NUM_COMPONENTS,
base_job_name='pca', sagemaker_session=sagemaker_session)
invalid_hyperparameter_ranges = {'num_components': IntegerParameter(-1, 2)}
with pytest.raises(ValueError) as e:
HyperparameterTuner(estimator=pca, objective_metric_name=OBJECTIVE_METRIC_NAME,
hyperparameter_ranges=invalid_hyperparameter_ranges, metric_definitions=METRIC_DEFINTIONS)
assert 'Value must be an integer greater than zero' in str(e)
def test_validate_parameter_ranges_string_value_validation_error(sagemaker_session):
pca = PCA(ROLE, TRAIN_INSTANCE_COUNT, TRAIN_INSTANCE_TYPE, NUM_COMPONENTS,
base_job_name='pca', sagemaker_session=sagemaker_session)
invalid_hyperparameter_ranges = {'algorithm_mode': CategoricalParameter([0, 5])}
with pytest.raises(ValueError) as e:
HyperparameterTuner(estimator=pca, objective_metric_name=OBJECTIVE_METRIC_NAME,
hyperparameter_ranges=invalid_hyperparameter_ranges, metric_definitions=METRIC_DEFINTIONS)
assert 'Value must be one of "regular" and "randomized"' in str(e)
def test_transfer_learning_tuner(sagemaker_session):
job_details = copy.deepcopy(TUNING_JOB_DETAILS)
sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job',
return_value=job_details)
tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session)
parent_tuner = tuner.transfer_learning_tuner(additional_parents={"p1", "p2"})
assert parent_tuner.warm_start_config.type == WarmStartTypes.TRANSFER_LEARNING
assert parent_tuner.warm_start_config.parents == {tuner.latest_tuning_job.name, "p1", "p2"}
assert parent_tuner.estimator == tuner.estimator
def test_attach_with_warm_start_config(sagemaker_session):
warm_start_config = WarmStartConfig(warm_start_type=WarmStartTypes.TRANSFER_LEARNING, parents={"p1", "p2"})
job_details = copy.deepcopy(TUNING_JOB_DETAILS)
job_details["WarmStartConfig"] = warm_start_config.to_input_req()
sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job',
return_value=job_details)
tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session)
assert tuner.warm_start_config.type == warm_start_config.type
assert tuner.warm_start_config.parents == warm_start_config.parents
def test_attach_tuning_job_with_estimator_from_hyperparameters_with_early_stopping(sagemaker_session):
job_details = copy.deepcopy(TUNING_JOB_DETAILS)
job_details['HyperParameterTuningJobConfig']['TrainingJobEarlyStoppingType'] = 'Auto'
sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job',
return_value=job_details)
tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session)
assert tuner.latest_tuning_job.name == JOB_NAME
assert tuner.early_stopping_type == 'Auto'
assert isinstance(tuner.estimator, PCA)
def get_sagemaker_tuner(self, **kwargs):
return HyperparameterTuner(
base_tuning_job_name=self.get_tuning_job_name(),
estimator=self.get_sagemaker_estimator(),
objective_metric_name=self.get_tuner_objective_metric_name(),
objective_type=kwargs.get("objective_type", "Minimize"),
hyperparameter_ranges=kwargs["hyperparameter_ranges"],
metric_definitions=self.get_tuner_metric_definitions(),
max_jobs=kwargs.get("max_jobs", 1),
max_parallel_jobs=kwargs.get("max_parallel_jobs", 1),
)
def _attach_tuner(sagemaker_session, tuning_job_name):
print("Attaching hyperparameter tuning job {} to a new tuner instance".
format(tuning_job_name))
return HyperparameterTuner.attach(
tuning_job_name,
sagemaker_session=sagemaker_session,
estimator_cls={
ESTIMATOR_FM: "sagemaker.estimator.Estimator",
ESTIMATOR_KNN: "sagemaker.estimator.Estimator",
},
)
def test_tuning_tf(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, 'iris', 'iris-dnn-classifier.py')
estimator = TensorFlow(entry_point=script_path,
role='SageMakerRole',
training_steps=1,
evaluation_steps=1,
hyperparameters={'input_tensor_name': 'inputs'},
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session,
base_job_name='tune-tf')
inputs = sagemaker_session.upload_data(
path=DATA_PATH, key_prefix='integ-test-data/tf_iris')
hyperparameter_ranges = {
'learning_rate': ContinuousParameter(0.05, 0.2)
}
objective_metric_name = 'loss'
metric_definitions = [{'Name': 'loss', 'Regex': 'loss = ([0-9\\.]+)'}]
tuner = HyperparameterTuner(estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
objective_type='Minimize',
max_jobs=2,
max_parallel_jobs=2)
tuner.fit(inputs)
print('Started hyperparameter tuning job with name:' +
tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job,
sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
features = [6.4, 3.2, 4.5, 1.5]
dict_result = predictor.predict({'inputs': features})
print('predict result: {}'.format(dict_result))
list_result = predictor.predict(features)
print('predict result: {}'.format(list_result))
assert dict_result == list_result
def test_tuning_kmeans(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}
# Load the data into memory as numpy arrays
with gzip.open(data_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
kmeans = KMeans(role='SageMakerRole', train_instance_count=1,
train_instance_type='ml.c4.xlarge',
k=10, sagemaker_session=sagemaker_session, base_job_name='tk',
output_path='s3://{}/'.format(sagemaker_session.default_bucket()))
# set kmeans specific hp
kmeans.init_method = 'random'
kmeans.max_iterators = 1
kmeans.tol = 1
kmeans.num_trials = 1
kmeans.local_init_method = 'kmeans++'
kmeans.half_life_time_size = 1
kmeans.epochs = 1
records = kmeans.record_set(train_set[0][:100])
test_records = kmeans.record_set(train_set[0][:100], channel='test')
# specify which hp you want to optimize over
hyperparameter_ranges = {'extra_center_factor': IntegerParameter(1, 10),
'mini_batch_size': IntegerParameter(10, 100),
'epochs': IntegerParameter(1, 2),
'init_method': CategoricalParameter(['kmeans++', 'random'])}
objective_metric_name = 'test:msd'
tuner = HyperparameterTuner(estimator=kmeans, objective_metric_name=objective_metric_name,
hyperparameter_ranges=hyperparameter_ranges, objective_type='Minimize', max_jobs=2,
max_parallel_jobs=2)
tuner.fit([records, test_records])
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
result = predictor.predict(train_set[0][:10])
assert len(result) == 10
for record in result:
assert record.label['closest_cluster'] is not None
assert record.label['distance_to_cluster'] is not None
def test_tuning_chainer(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, 'chainer_mnist', 'mnist.py')
data_path = os.path.join(DATA_DIR, 'chainer_mnist')
estimator = Chainer(entry_point=script_path,
role='SageMakerRole',
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session,
hyperparameters={'epochs': 1})
train_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'train'),
key_prefix='integ-test-data/chainer_mnist/train')
test_input = estimator.sagemaker_session.upload_data(path=os.path.join(data_path, 'test'),
key_prefix='integ-test-data/chainer_mnist/test')
hyperparameter_ranges = {'alpha': ContinuousParameter(0.001, 0.005)}
objective_metric_name = 'Validation-accuracy'
metric_definitions = [
{'Name': 'Validation-accuracy', 'Regex': '\[J1\s+\d\.\d+\s+\d\.\d+\s+\d\.\d+\s+(\d\.\d+)'}]
tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions,
max_jobs=2, max_parallel_jobs=2)
tuner.fit({'train': train_input, 'test': test_input})
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
batch_size = 100
data = np.zeros((batch_size, 784), dtype='float32')
output = predictor.predict(data)
assert len(output) == batch_size
data = np.zeros((batch_size, 1, 28, 28), dtype='float32')
output = predictor.predict(data)
assert len(output) == batch_size
data = np.zeros((batch_size, 28, 28), dtype='float32')
output = predictor.predict(data)
assert len(output) == batch_size
def test_tuning_lda(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
data_path = os.path.join(DATA_DIR, 'lda')
data_filename = 'nips-train_1.pbr'
with open(os.path.join(data_path, data_filename), 'rb') as f:
all_records = read_records(f)
# all records must be same
feature_num = int(all_records[0].features['values'].float32_tensor.shape[0])
lda = LDA(role='SageMakerRole', train_instance_type='ml.c4.xlarge', num_topics=10,
sagemaker_session=sagemaker_session, base_job_name='test-lda')
record_set = prepare_record_set_from_local_files(data_path, lda.data_location,
len(all_records), feature_num, sagemaker_session)
test_record_set = prepare_record_set_from_local_files(data_path, lda.data_location,
len(all_records), feature_num, sagemaker_session)
test_record_set.channel = 'test'
# specify which hp you want to optimize over
hyperparameter_ranges = {'alpha0': ContinuousParameter(1, 10),
'num_topics': IntegerParameter(1, 2)}
objective_metric_name = 'test:pwll'
tuner = HyperparameterTuner(estimator=lda, objective_metric_name=objective_metric_name,
hyperparameter_ranges=hyperparameter_ranges, objective_type='Maximize', max_jobs=2,
max_parallel_jobs=2)
tuner.fit([record_set, test_record_set], mini_batch_size=1)
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
predict_input = np.random.rand(1, feature_num)
result = predictor.predict(predict_input)
assert len(result) == 1
for record in result:
assert record.label['topic_mixture'] is not None
def test_tuning_tf(sagemaker_session):
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
script_path = os.path.join(DATA_DIR, 'iris', 'iris-dnn-classifier.py')
estimator = TensorFlow(entry_point=script_path,
role='SageMakerRole',
training_steps=1,
evaluation_steps=1,
hyperparameters={'input_tensor_name': 'inputs'},
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session,
base_job_name='tune-tf')
inputs = sagemaker_session.upload_data(path=DATA_PATH, key_prefix='integ-test-data/tf_iris')
hyperparameter_ranges = {'learning_rate': ContinuousParameter(0.05, 0.2)}
objective_metric_name = 'loss'
metric_definitions = [{'Name': 'loss', 'Regex': 'loss = ([0-9\\.]+)'}]
tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions,
objective_type='Minimize', max_jobs=2, max_parallel_jobs=2)
tuner.fit(inputs)
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
features = [6.4, 3.2, 4.5, 1.5]
dict_result = predictor.predict({'inputs': features})
print('predict result: {}'.format(dict_result))
list_result = predictor.predict(features)
print('predict result: {}'.format(list_result))
assert dict_result == list_result
def test_attach_tuning_job_with_estimator_from_hyperparameters(sagemaker_session):
job_details = copy.deepcopy(TUNING_JOB_DETAILS)
sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job',
return_value=job_details)
tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session)
assert tuner.latest_tuning_job.name == JOB_NAME
assert tuner.objective_metric_name == OBJECTIVE_METRIC_NAME
assert tuner.max_jobs == 1
assert tuner.max_parallel_jobs == 1
assert tuner.metric_definitions == METRIC_DEFINTIONS
assert tuner.strategy == 'Bayesian'
assert tuner.objective_type == 'Minimize'
assert isinstance(tuner.estimator, PCA)
assert tuner.estimator.role == ROLE
assert tuner.estimator.train_instance_count == 1
assert tuner.estimator.train_max_run == 24 * 60 * 60
assert tuner.estimator.input_mode == 'File'
assert tuner.estimator.output_path == BUCKET_NAME
assert tuner.estimator.output_kms_key == ''
assert '_tuning_objective_metric' not in tuner.estimator.hyperparameters()
assert tuner.estimator.hyperparameters()['num_components'] == '1'
def test_attach_tuning_pytorch(sagemaker_session):
mnist_dir = os.path.join(DATA_DIR, 'pytorch_mnist')
mnist_script = os.path.join(mnist_dir, 'mnist.py')
estimator = PyTorch(entry_point=mnist_script, role='SageMakerRole', train_instance_count=1,
train_instance_type='ml.c4.xlarge', sagemaker_session=sagemaker_session)
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
objective_metric_name = 'evaluation-accuracy'
metric_definitions = [{'Name': 'evaluation-accuracy', 'Regex': 'Overall test accuracy: (\d+)'}]
hyperparameter_ranges = {'batch-size': IntegerParameter(50, 100)}
tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions,
max_jobs=2, max_parallel_jobs=2)
training_data = estimator.sagemaker_session.upload_data(path=os.path.join(mnist_dir, 'training'),
key_prefix='integ-test-data/pytorch_mnist/training')
tuner.fit({'training': training_data})
tuning_job_name = tuner.latest_tuning_job.name
print('Started hyperparameter tuning job with name:' + tuning_job_name)
time.sleep(15)
tuner.wait()
attached_tuner = HyperparameterTuner.attach(tuning_job_name, sagemaker_session=sagemaker_session)
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = attached_tuner.deploy(1, 'ml.c4.xlarge')
data = np.zeros(shape=(1, 1, 28, 28), dtype=np.float32)
predictor.predict(data)
batch_size = 100
data = np.random.rand(batch_size, 1, 28, 28).astype(np.float32)
output = predictor.predict(data)
assert output.shape == (batch_size, 10)
def test_attach_tuning_job_with_job_details(sagemaker_session):
job_details = copy.deepcopy(TUNING_JOB_DETAILS)
HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session, job_details=job_details)
sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job.assert_not_called
def test_tuning_byo_estimator(sagemaker_session):
"""Use Factorization Machines algorithm as an example here.
First we need to prepare data for training. We take standard data set, convert it to the
format that the algorithm can process and upload it to S3.
Then we create the Estimator and set hyperparamets as required by the algorithm.
Next, we can call fit() with path to the S3.
Later the trained model is deployed and prediction is called against the endpoint.
Default predictor is updated with json serializer and deserializer.
"""
image_name = registry(sagemaker_session.boto_session.region_name) + '/factorization-machines:1'
training_data_path = os.path.join(DATA_DIR, 'dummy_tensor')
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}
with gzip.open(data_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
prefix = 'test_byo_estimator'
key = 'recordio-pb-data'
s3_train_data = sagemaker_session.upload_data(path=training_data_path,
key_prefix=os.path.join(prefix, 'train', key))
estimator = Estimator(image_name=image_name,
role='SageMakerRole', train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session, base_job_name='test-byo')
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type='binary_classifier')
hyperparameter_ranges = {'mini_batch_size': IntegerParameter(100, 200)}
tuner = HyperparameterTuner(estimator=estimator, base_tuning_job_name='byo',
objective_metric_name='test:binary_classification_accuracy',
hyperparameter_ranges=hyperparameter_ranges,
max_jobs=2, max_parallel_jobs=2)
tuner.fit({'train': s3_train_data, 'test': s3_train_data}, include_cls_metadata=False)
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.m4.xlarge', endpoint_name=best_training_job)
predictor.serializer = _fm_serializer
predictor.content_type = 'application/json'
predictor.deserializer = json_deserializer
result = predictor.predict(train_set[0][:10])
assert len(result['predictions']) == 10
for prediction in result['predictions']:
assert prediction['score'] is not None
