def test_fit_pca(sagemaker_session, tuner):
pca = PCA(ROLE, TRAIN_INSTANCE_COUNT, TRAIN_INSTANCE_TYPE, NUM_COMPONENTS,
base_job_name='pca', sagemaker_session=sagemaker_session)
pca.algorithm_mode = 'randomized'
pca.subtract_mean = True
pca.extra_components = 5
tuner.estimator = pca
tags = [{'Name': 'some-tag-without-a-value'}]
tuner.tags = tags
hyperparameter_ranges = {'num_components': IntegerParameter(2, 4),
'algorithm_mode': CategoricalParameter(['regular', 'randomized'])}
tuner._hyperparameter_ranges = hyperparameter_ranges
records = RecordSet(s3_data=INPUTS, num_records=1, feature_dim=1)
tuner.fit(records, mini_batch_size=9999)
_, _, tune_kwargs = sagemaker_session.tune.mock_calls[0]
assert len(tune_kwargs['static_hyperparameters']) == 4
assert tune_kwargs['static_hyperparameters']['extra_components'] == '5'
assert len(tune_kwargs['parameter_ranges']['IntegerParameterRanges']) == 1
assert tune_kwargs['job_name'].startswith('pca')
assert tune_kwargs['tags'] == tags
assert tuner.estimator.mini_batch_size == 9999
def _get_categorical_parameter(self, data):
if "values" not in data:
raise DescriptorError(
'The "values" attribute of a categorical parameter is required'
)
return CategoricalParameter(values=data["values"])
def hyperparameter_ranges():
return {
"extra_center_factor": IntegerParameter(1, 10),
"mini_batch_size": IntegerParameter(10, 100),
"epochs": IntegerParameter(1, 2),
"init_method": CategoricalParameter(["kmeans++", "random"]),
}
def hyperparameter_ranges():
return {
'extra_center_factor': IntegerParameter(1, 10),
'mini_batch_size': IntegerParameter(10, 100),
'epochs': IntegerParameter(1, 2),
'init_method': CategoricalParameter(['kmeans++', 'random'])
}
def test_validate_parameter_ranges_string_value_validation_error(sagemaker_session):
pca = PCA(ROLE, TRAIN_INSTANCE_COUNT, TRAIN_INSTANCE_TYPE, NUM_COMPONENTS,
base_job_name='pca', sagemaker_session=sagemaker_session)
invalid_hyperparameter_ranges = {'algorithm_mode': CategoricalParameter([0, 5])}
with pytest.raises(ValueError) as e:
HyperparameterTuner(estimator=pca, objective_metric_name=OBJECTIVE_METRIC_NAME,
hyperparameter_ranges=invalid_hyperparameter_ranges, metric_definitions=METRIC_DEFINTIONS)
assert 'Value must be one of "regular" and "randomized"' in str(e)
def test_tuning_kmeans(sagemaker_session):
with timeout(minutes=20):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}
# Load the data into memory as numpy arrays
with gzip.open(data_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
kmeans = KMeans(role='SageMakerRole', train_instance_count=1,
train_instance_type='ml.c4.xlarge',
k=10, sagemaker_session=sagemaker_session, base_job_name='tk',
output_path='s3://{}/'.format(sagemaker_session.default_bucket()))
# set kmeans specific hp
kmeans.init_method = 'random'
kmeans.max_iterators = 1
kmeans.tol = 1
kmeans.num_trials = 1
kmeans.local_init_method = 'kmeans++'
kmeans.half_life_time_size = 1
kmeans.epochs = 1
records = kmeans.record_set(train_set[0][:100])
test_records = kmeans.record_set(train_set[0][:100], channel='test')
# specify which hp you want to optimize over
hyperparameter_ranges = {'extra_center_factor': IntegerParameter(1, 10),
'mini_batch_size': IntegerParameter(10, 100),
'epochs': IntegerParameter(1, 2),
'init_method': CategoricalParameter(['kmeans++', 'random'])}
objective_metric_name = 'test:msd'
tuner = HyperparameterTuner(estimator=kmeans, objective_metric_name=objective_metric_name,
hyperparameter_ranges=hyperparameter_ranges, objective_type='Minimize', max_jobs=2,
max_parallel_jobs=2)
tuner.fit([records, test_records])
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.c4.xlarge')
result = predictor.predict(train_set[0][:10])
assert len(result) == 10
for record in result:
assert record.label['closest_cluster'] is not None
assert record.label['distance_to_cluster'] is not None
def test_categorical_parameter_list_ranges():
cat_param = CategoricalParameter([1, 10])
ranges = cat_param.as_tuning_range('some')
assert len(ranges.keys()) == 2
assert ranges['Name'] == 'some'
assert ranges['Values'] == ['1', '10']
def test_categorical_parameter_list():
cat_param = CategoricalParameter(['a', 'z'])
assert isinstance(cat_param, _ParameterRange)
assert cat_param.__name__ is 'Categorical'
BUCKET_NAME = 'Some-Bucket'
ROLE = 'myrole'
IMAGE_NAME = 'image'
TRAIN_INSTANCE_COUNT = 1
TRAIN_INSTANCE_TYPE = 'ml.c4.xlarge'
NUM_COMPONENTS = 5
SCRIPT_NAME = 'my_script.py'
FRAMEWORK_VERSION = '1.0.0'
INPUTS = 's3://mybucket/train'
OBJECTIVE_METRIC_NAME = 'mock_metric'
HYPERPARAMETER_RANGES = {'validated': ContinuousParameter(0, 5),
'elizabeth': IntegerParameter(0, 5),
'blank': CategoricalParameter([0, 5])}
METRIC_DEFINTIONS = 'mock_metric_definitions'
TUNING_JOB_DETAILS = {
'HyperParameterTuningJobConfig': {
'ResourceLimits': {
'MaxParallelTrainingJobs': 1,
'MaxNumberOfTrainingJobs': 1
},
'HyperParameterTuningJobObjective': {
'MetricName': OBJECTIVE_METRIC_NAME,
'Type': 'Minimize'
},
'Strategy': 'Bayesian',
'ParameterRanges': {
'CategoricalParameterRanges': [],
#Set up the job
from sagemaker.pytorch import PyTorch
estimator = PyTorch(entry_point="mnist.py",
role=role,
framework_version='1.4.0',
train_instance_count=1,
train_instance_type='ml.m4.xlarge',
hyperparameters={
'epochs': 6,
'backend': 'gloo'
})
hyperparameter_ranges = {'lr': ContinuousParameter(0.001, 0.1),'batch-size': CategoricalParameter([32,64,128,256,512])}
objective_metric_name = 'average test loss'
objective_type = 'Minimize'
metric_definitions = [{'Name': 'average test loss',
'Regex': 'Test set: Average loss: ([0-9\\.]+)'}]
tuner = HyperparameterTuner(estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
max_jobs=9,
max_parallel_jobs=3,
region = boto3.Session().region_name
train_data_location = 's3://sagemaker-sample-data-{}/mxnet/mnist/train'.format(
region)
test_data_location = 's3://sagemaker-sample-data-{}/mxnet/mnist/test'.format(
region)
estimator = MXNet(entry_point='mnist.py',
role=role,
train_instance_count=1,
train_instance_type='ml.m4.xlarge',
sagemaker_session=sagemaker.Session(),
base_job_name='DEMO-hpo-mxnet',
hyperparameters={'batch_size': 100})
hyperparameter_ranges = {
'optimizer': CategoricalParameter(['sgd', 'Adam']),
'learning_rate': ContinuousParameter(0.01, 0.2),
'num_epoch': IntegerParameter(10, 50)
}
objective_metric_name = 'Validation-accuracy'
metric_definitions = [{
'Name': 'Validation-accuracy',
'Regex': 'Validation-accuracy=([0-9\\.]+)'
}]
tuner = HyperparameterTuner(estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
max_jobs=9,
def test_categorical_parameter_value_ranges():
cat_param = CategoricalParameter('a')
ranges = cat_param.as_tuning_range('some')
assert len(ranges.keys()) == 2
assert ranges['Name'] == 'some'
assert ranges['Values'] == ['a']
def test_categorical_parameter_list_ranges():
cat_param = CategoricalParameter([1, 10])
ranges = cat_param.as_tuning_range('some')
assert len(ranges.keys()) == 2
assert ranges['Name'] == 'some'
assert ranges['Values'] == ['1', '10']
entry_point_path = os.path.join(DIR_PATH, "fashion_mnist.py")
estimator = PyTorch(entry_point=entry_point_path,
role=role,
sagemaker_session=sagemaker_session,
framework_version='1.0.0.dev',
train_instance_count=1,
train_instance_type='ml.m4.xlarge',
hyperparameters={
'epochs': 6,
'backend': 'gloo'
})
hyperparameter_ranges = {
'lr': ContinuousParameter(0.001, 0.1),
'batch-size': CategoricalParameter([32, 64, 128, 256, 512])
}
objective_metric_name = 'average test loss'
objective_type = 'Minimize'
metric_definitions = [{
'Name': 'average test loss',
'Regex': 'Test set: Average loss: ([0-9\\.]+)'
}]
tuner = HyperparameterTuner(estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
max_jobs=3,
max_parallel_jobs=3,
def test_categorical_parameter_value():
cat_param = CategoricalParameter('a')
assert isinstance(cat_param, _ParameterRange)
estimator = PyTorch(entry_point="cifar10.py",
source_dir=os.getcwd() + "/source",
role=role,
framework_version='1.0.0.dev',
train_instance_count=1,
train_instance_type='ml.c5.xlarge',
hyperparameters={
'epochs': 50,
'momentum': 0.9
})
hyperparameter_ranges = {
'lr': ContinuousParameter(0.0001, 0.001),
'hidden_nodes': IntegerParameter(20, 100),
'batch_size': CategoricalParameter([128, 256, 512]),
'conv1_channels': CategoricalParameter([32, 64, 128]),
'conv2_channels': CategoricalParameter([64, 128, 256, 512]),
}
objective_metric_name = 'average test accuracy'
objective_type = 'Maximize'
metric_definitions = [{
'Name': 'average test accuracy',
'Regex': 'Test Accuracy: ([0-9\\.]+)'
}]
tuner = HyperparameterTuner(estimator,
objective_metric_name,
hyperparameter_ranges,
metric_definitions,
def test_categorical_parameter_value_ranges():
cat_param = CategoricalParameter('a')
ranges = cat_param.as_tuning_range('some')
assert len(ranges.keys()) == 2
assert ranges['Name'] == 'some'
assert ranges['Values'] == ['a']
def getter(self, attr: str) -> Dict[str, Any]:
data = {
'tfrecord_processing': {
'endpoint': ['python3', 'criteo_ads_data/run_processing.py'],
'inputs': [
ProcessingInput(
source='s3://criteo-ads-data/prod/train_csv',
destination='/opt/ml/processing/input',
s3_data_distribution_type='ShardedByS3Key',
)
],
'outputs': [
ProcessingOutput(
source='/opt/ml/processing/output',
destination=
's3://criteo-ads-data/prod/train_tfrecord_gz',
)
],
'arguments': [
'--input_path=/opt/ml/processing/input',
'--output_path=/opt/ml/processing/output',
],
'sm_config':
SagemakerProcessingConfig(
project_name=self.project_name,
env=self.env,
region_name=self.region_name,
current_time=self.current_time,
sm_instance_type='ml.c5.2xlarge',
sm_instance_count=20,
sm_volumesize=100,
max_run=1 * 60 * 60,
)
},
'layer_processing': {
'endpoint':
['python3', 'criteo_ads_data/run_processing_layer.py'],
'inputs': [
ProcessingInput(
source=
's3://criteo-ads-data/prod/train_tfrecord_gz/train',
destination='/opt/ml/processing/input',
s3_data_distribution_type='FullyReplicated',
)
],
'outputs': [
ProcessingOutput(
source='/opt/ml/processing/output',
destination='s3://criteo-ads-data/prod/proc_layer',
)
],
'arguments': [
'--input_path=/opt/ml/processing/input',
'--output_path=/opt/ml/processing/output',
],
'sm_config':
SagemakerProcessingConfig(
project_name=self.project_name,
env=self.env,
region_name=self.region_name,
current_time=self.current_time,
sm_instance_type='ml.c5.9xlarge',
sm_instance_count=1,
sm_volumesize=100,
max_run=24 * 60 * 60,
)
},
'estimator': {
'sm_input': {
'train':
TrainingInput(
s3_data=
's3://criteo-ads-data/prod/train_tfrecord_100000_gz/train',
distribution='FullyReplicated',
),
'test':
TrainingInput(
s3_data=
's3://criteo-ads-data/prod/train_tfrecord_100000_gz/test',
distribution='FullyReplicated',
),
'layer':
TrainingInput(
s3_data='s3://criteo-ads-data/prod/proc_layer_100000',
distribution='FullyReplicated',
),
},
'shared_hyperparameters': {
'tf_logs_path': self.tf_logs_path,
'batch_size': 512,
},
'sm_config':
SagemakerTrainingConfig(
project_name=self.project_name,
env=self.env,
region_name=self.region_name,
current_time=self.current_time,
sm_instance_type='ml.c5.2xlarge',
sm_instance_count=1,
sm_volumesize=300,
max_run=1 * 24 * 60 * 60,
)
},
'hparam_tuning': {
'objective_metric_name':
'validation:loss',
'metric_definitions': [
{
'Name': 'train:loss',
'Regex': '.*loss: ([0-9\\.]+) - auc: [0-9\\.]+.*'
},
{
'Name': 'train:auc',
'Regex': '.*loss: [0-9\\.]+ - auc: ([0-9\\.]+).*'
},
{
'Name':
'validation:loss',
'Regex':
'.*step - loss: [0-9\\.]+ - auc: [0-9\\.]+ - val_loss: ([0-9\\.]+) - val_auc: [0-9\\.]+.*'
},
{
'Name':
'validation:auc',
'Regex':
'.*step - loss: [0-9\\.]+ - auc: [0-9\\.]+ - val_loss: [0-9\\.]+ - val_auc: ([0-9\\.]+).*'
},
],
'hyperparameter_ranges': {
'epochs': IntegerParameter(1, 50),
'batch_size': CategoricalParameter([64, 128, 256, 512])
},
'objective_type':
'Minimize',
'max_jobs':
5,
'max_parallel_jobs':
5,
},
}
return data.get(attr)
