def test_fit_pca(sagemaker_session, tuner): pca = PCA(ROLE, TRAIN_INSTANCE_COUNT, TRAIN_INSTANCE_TYPE, NUM_COMPONENTS, base_job_name='pca', sagemaker_session=sagemaker_session) pca.algorithm_mode = 'randomized' pca.subtract_mean = True pca.extra_components = 5 tuner.estimator = pca tags = [{'Name': 'some-tag-without-a-value'}] tuner.tags = tags hyperparameter_ranges = {'num_components': IntegerParameter(2, 4), 'algorithm_mode': CategoricalParameter(['regular', 'randomized'])} tuner._hyperparameter_ranges = hyperparameter_ranges records = RecordSet(s3_data=INPUTS, num_records=1, feature_dim=1) tuner.fit(records, mini_batch_size=9999) _, _, tune_kwargs = sagemaker_session.tune.mock_calls[0] assert len(tune_kwargs['static_hyperparameters']) == 4 assert tune_kwargs['static_hyperparameters']['extra_components'] == '5' assert len(tune_kwargs['parameter_ranges']['IntegerParameterRanges']) == 1 assert tune_kwargs['job_name'].startswith('pca') assert tune_kwargs['tags'] == tags assert tuner.estimator.mini_batch_size == 9999
def _get_categorical_parameter(self, data): if "values" not in data: raise DescriptorError( 'The "values" attribute of a categorical parameter is required' ) return CategoricalParameter(values=data["values"])
def hyperparameter_ranges(): return { "extra_center_factor": IntegerParameter(1, 10), "mini_batch_size": IntegerParameter(10, 100), "epochs": IntegerParameter(1, 2), "init_method": CategoricalParameter(["kmeans++", "random"]), }
def hyperparameter_ranges(): return { 'extra_center_factor': IntegerParameter(1, 10), 'mini_batch_size': IntegerParameter(10, 100), 'epochs': IntegerParameter(1, 2), 'init_method': CategoricalParameter(['kmeans++', 'random']) }
def test_validate_parameter_ranges_string_value_validation_error(sagemaker_session): pca = PCA(ROLE, TRAIN_INSTANCE_COUNT, TRAIN_INSTANCE_TYPE, NUM_COMPONENTS, base_job_name='pca', sagemaker_session=sagemaker_session) invalid_hyperparameter_ranges = {'algorithm_mode': CategoricalParameter([0, 5])} with pytest.raises(ValueError) as e: HyperparameterTuner(estimator=pca, objective_metric_name=OBJECTIVE_METRIC_NAME, hyperparameter_ranges=invalid_hyperparameter_ranges, metric_definitions=METRIC_DEFINTIONS) assert 'Value must be one of "regular" and "randomized"' in str(e)
def test_tuning_kmeans(sagemaker_session): with timeout(minutes=20): data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz') pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'} # Load the data into memory as numpy arrays with gzip.open(data_path, 'rb') as f: train_set, _, _ = pickle.load(f, **pickle_args) kmeans = KMeans(role='SageMakerRole', train_instance_count=1, train_instance_type='ml.c4.xlarge', k=10, sagemaker_session=sagemaker_session, base_job_name='tk', output_path='s3://{}/'.format(sagemaker_session.default_bucket())) # set kmeans specific hp kmeans.init_method = 'random' kmeans.max_iterators = 1 kmeans.tol = 1 kmeans.num_trials = 1 kmeans.local_init_method = 'kmeans++' kmeans.half_life_time_size = 1 kmeans.epochs = 1 records = kmeans.record_set(train_set[0][:100]) test_records = kmeans.record_set(train_set[0][:100], channel='test') # specify which hp you want to optimize over hyperparameter_ranges = {'extra_center_factor': IntegerParameter(1, 10), 'mini_batch_size': IntegerParameter(10, 100), 'epochs': IntegerParameter(1, 2), 'init_method': CategoricalParameter(['kmeans++', 'random'])} objective_metric_name = 'test:msd' tuner = HyperparameterTuner(estimator=kmeans, objective_metric_name=objective_metric_name, hyperparameter_ranges=hyperparameter_ranges, objective_type='Minimize', max_jobs=2, max_parallel_jobs=2) tuner.fit([records, test_records]) print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name) time.sleep(15) tuner.wait() best_training_job = tuner.best_training_job() with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session): predictor = tuner.deploy(1, 'ml.c4.xlarge') result = predictor.predict(train_set[0][:10]) assert len(result) == 10 for record in result: assert record.label['closest_cluster'] is not None assert record.label['distance_to_cluster'] is not None
def test_categorical_parameter_list_ranges(): cat_param = CategoricalParameter([1, 10]) ranges = cat_param.as_tuning_range('some') assert len(ranges.keys()) == 2 assert ranges['Name'] == 'some' assert ranges['Values'] == ['1', '10']
def test_categorical_parameter_list(): cat_param = CategoricalParameter(['a', 'z']) assert isinstance(cat_param, _ParameterRange) assert cat_param.__name__ is 'Categorical'
BUCKET_NAME = 'Some-Bucket' ROLE = 'myrole' IMAGE_NAME = 'image' TRAIN_INSTANCE_COUNT = 1 TRAIN_INSTANCE_TYPE = 'ml.c4.xlarge' NUM_COMPONENTS = 5 SCRIPT_NAME = 'my_script.py' FRAMEWORK_VERSION = '1.0.0' INPUTS = 's3://mybucket/train' OBJECTIVE_METRIC_NAME = 'mock_metric' HYPERPARAMETER_RANGES = {'validated': ContinuousParameter(0, 5), 'elizabeth': IntegerParameter(0, 5), 'blank': CategoricalParameter([0, 5])} METRIC_DEFINTIONS = 'mock_metric_definitions' TUNING_JOB_DETAILS = { 'HyperParameterTuningJobConfig': { 'ResourceLimits': { 'MaxParallelTrainingJobs': 1, 'MaxNumberOfTrainingJobs': 1 }, 'HyperParameterTuningJobObjective': { 'MetricName': OBJECTIVE_METRIC_NAME, 'Type': 'Minimize' }, 'Strategy': 'Bayesian', 'ParameterRanges': { 'CategoricalParameterRanges': [],
#Set up the job from sagemaker.pytorch import PyTorch estimator = PyTorch(entry_point="mnist.py", role=role, framework_version='1.4.0', train_instance_count=1, train_instance_type='ml.m4.xlarge', hyperparameters={ 'epochs': 6, 'backend': 'gloo' }) hyperparameter_ranges = {'lr': ContinuousParameter(0.001, 0.1),'batch-size': CategoricalParameter([32,64,128,256,512])} objective_metric_name = 'average test loss' objective_type = 'Minimize' metric_definitions = [{'Name': 'average test loss', 'Regex': 'Test set: Average loss: ([0-9\\.]+)'}] tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions, max_jobs=9, max_parallel_jobs=3,
region = boto3.Session().region_name train_data_location = 's3://sagemaker-sample-data-{}/mxnet/mnist/train'.format( region) test_data_location = 's3://sagemaker-sample-data-{}/mxnet/mnist/test'.format( region) estimator = MXNet(entry_point='mnist.py', role=role, train_instance_count=1, train_instance_type='ml.m4.xlarge', sagemaker_session=sagemaker.Session(), base_job_name='DEMO-hpo-mxnet', hyperparameters={'batch_size': 100}) hyperparameter_ranges = { 'optimizer': CategoricalParameter(['sgd', 'Adam']), 'learning_rate': ContinuousParameter(0.01, 0.2), 'num_epoch': IntegerParameter(10, 50) } objective_metric_name = 'Validation-accuracy' metric_definitions = [{ 'Name': 'Validation-accuracy', 'Regex': 'Validation-accuracy=([0-9\\.]+)' }] tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions, max_jobs=9,
def test_categorical_parameter_value_ranges(): cat_param = CategoricalParameter('a') ranges = cat_param.as_tuning_range('some') assert len(ranges.keys()) == 2 assert ranges['Name'] == 'some' assert ranges['Values'] == ['a']
entry_point_path = os.path.join(DIR_PATH, "fashion_mnist.py") estimator = PyTorch(entry_point=entry_point_path, role=role, sagemaker_session=sagemaker_session, framework_version='1.0.0.dev', train_instance_count=1, train_instance_type='ml.m4.xlarge', hyperparameters={ 'epochs': 6, 'backend': 'gloo' }) hyperparameter_ranges = { 'lr': ContinuousParameter(0.001, 0.1), 'batch-size': CategoricalParameter([32, 64, 128, 256, 512]) } objective_metric_name = 'average test loss' objective_type = 'Minimize' metric_definitions = [{ 'Name': 'average test loss', 'Regex': 'Test set: Average loss: ([0-9\\.]+)' }] tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions, max_jobs=3, max_parallel_jobs=3,
def test_categorical_parameter_value(): cat_param = CategoricalParameter('a') assert isinstance(cat_param, _ParameterRange)
estimator = PyTorch(entry_point="cifar10.py", source_dir=os.getcwd() + "/source", role=role, framework_version='1.0.0.dev', train_instance_count=1, train_instance_type='ml.c5.xlarge', hyperparameters={ 'epochs': 50, 'momentum': 0.9 }) hyperparameter_ranges = { 'lr': ContinuousParameter(0.0001, 0.001), 'hidden_nodes': IntegerParameter(20, 100), 'batch_size': CategoricalParameter([128, 256, 512]), 'conv1_channels': CategoricalParameter([32, 64, 128]), 'conv2_channels': CategoricalParameter([64, 128, 256, 512]), } objective_metric_name = 'average test accuracy' objective_type = 'Maximize' metric_definitions = [{ 'Name': 'average test accuracy', 'Regex': 'Test Accuracy: ([0-9\\.]+)' }] tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions,
def getter(self, attr: str) -> Dict[str, Any]: data = { 'tfrecord_processing': { 'endpoint': ['python3', 'criteo_ads_data/run_processing.py'], 'inputs': [ ProcessingInput( source='s3://criteo-ads-data/prod/train_csv', destination='/opt/ml/processing/input', s3_data_distribution_type='ShardedByS3Key', ) ], 'outputs': [ ProcessingOutput( source='/opt/ml/processing/output', destination= 's3://criteo-ads-data/prod/train_tfrecord_gz', ) ], 'arguments': [ '--input_path=/opt/ml/processing/input', '--output_path=/opt/ml/processing/output', ], 'sm_config': SagemakerProcessingConfig( project_name=self.project_name, env=self.env, region_name=self.region_name, current_time=self.current_time, sm_instance_type='ml.c5.2xlarge', sm_instance_count=20, sm_volumesize=100, max_run=1 * 60 * 60, ) }, 'layer_processing': { 'endpoint': ['python3', 'criteo_ads_data/run_processing_layer.py'], 'inputs': [ ProcessingInput( source= 's3://criteo-ads-data/prod/train_tfrecord_gz/train', destination='/opt/ml/processing/input', s3_data_distribution_type='FullyReplicated', ) ], 'outputs': [ ProcessingOutput( source='/opt/ml/processing/output', destination='s3://criteo-ads-data/prod/proc_layer', ) ], 'arguments': [ '--input_path=/opt/ml/processing/input', '--output_path=/opt/ml/processing/output', ], 'sm_config': SagemakerProcessingConfig( project_name=self.project_name, env=self.env, region_name=self.region_name, current_time=self.current_time, sm_instance_type='ml.c5.9xlarge', sm_instance_count=1, sm_volumesize=100, max_run=24 * 60 * 60, ) }, 'estimator': { 'sm_input': { 'train': TrainingInput( s3_data= 's3://criteo-ads-data/prod/train_tfrecord_100000_gz/train', distribution='FullyReplicated', ), 'test': TrainingInput( s3_data= 's3://criteo-ads-data/prod/train_tfrecord_100000_gz/test', distribution='FullyReplicated', ), 'layer': TrainingInput( s3_data='s3://criteo-ads-data/prod/proc_layer_100000', distribution='FullyReplicated', ), }, 'shared_hyperparameters': { 'tf_logs_path': self.tf_logs_path, 'batch_size': 512, }, 'sm_config': SagemakerTrainingConfig( project_name=self.project_name, env=self.env, region_name=self.region_name, current_time=self.current_time, sm_instance_type='ml.c5.2xlarge', sm_instance_count=1, sm_volumesize=300, max_run=1 * 24 * 60 * 60, ) }, 'hparam_tuning': { 'objective_metric_name': 'validation:loss', 'metric_definitions': [ { 'Name': 'train:loss', 'Regex': '.*loss: ([0-9\\.]+) - auc: [0-9\\.]+.*' }, { 'Name': 'train:auc', 'Regex': '.*loss: [0-9\\.]+ - auc: ([0-9\\.]+).*' }, { 'Name': 'validation:loss', 'Regex': '.*step - loss: [0-9\\.]+ - auc: [0-9\\.]+ - val_loss: ([0-9\\.]+) - val_auc: [0-9\\.]+.*' }, { 'Name': 'validation:auc', 'Regex': '.*step - loss: [0-9\\.]+ - auc: [0-9\\.]+ - val_loss: [0-9\\.]+ - val_auc: ([0-9\\.]+).*' }, ], 'hyperparameter_ranges': { 'epochs': IntegerParameter(1, 50), 'batch_size': CategoricalParameter([64, 128, 256, 512]) }, 'objective_type': 'Minimize', 'max_jobs': 5, 'max_parallel_jobs': 5, }, } return data.get(attr)