def test_attach_tuning_job_with_estimator_from_kwarg(sagemaker_session): job_details = copy.deepcopy(TUNING_JOB_DETAILS) sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job', return_value=job_details) tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session, estimator_cls='sagemaker.estimator.Estimator') assert isinstance(tuner.estimator, Estimator)
def test_attach_tuning_job_with_estimator_from_hyperparameters(sagemaker_session): job_details = copy.deepcopy(TUNING_JOB_DETAILS) sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock( name="describe_tuning_job", return_value=job_details ) tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session) assert tuner.latest_tuning_job.name == JOB_NAME assert tuner.objective_metric_name == OBJECTIVE_METRIC_NAME assert tuner.max_jobs == 1 assert tuner.max_parallel_jobs == 1 assert tuner.metric_definitions == METRIC_DEFINITIONS assert tuner.strategy == "Bayesian" assert tuner.objective_type == "Minimize" assert tuner.early_stopping_type == "Off" assert isinstance(tuner.estimator, PCA) assert tuner.estimator.role == ROLE assert tuner.estimator.train_instance_count == 1 assert tuner.estimator.train_max_run == 24 * 60 * 60 assert tuner.estimator.input_mode == "File" assert tuner.estimator.output_path == BUCKET_NAME assert tuner.estimator.output_kms_key == "" assert "_tuning_objective_metric" not in tuner.estimator.hyperparameters() assert tuner.estimator.hyperparameters()["num_components"] == "10"
def test_attach_tuning_pytorch(sagemaker_session): mnist_dir = os.path.join(DATA_DIR, "pytorch_mnist") mnist_script = os.path.join(mnist_dir, "mnist.py") estimator = PyTorch( entry_point=mnist_script, role="SageMakerRole", train_instance_count=1, py_version=PYTHON_VERSION, train_instance_type="ml.c4.xlarge", sagemaker_session=sagemaker_session, ) with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES): objective_metric_name = "evaluation-accuracy" metric_definitions = [{ "Name": "evaluation-accuracy", "Regex": r"Overall test accuracy: (\d+)" }] hyperparameter_ranges = {"batch-size": IntegerParameter(50, 100)} tuner = HyperparameterTuner( estimator, objective_metric_name, hyperparameter_ranges, metric_definitions, max_jobs=2, max_parallel_jobs=2, early_stopping_type="Auto", ) training_data = estimator.sagemaker_session.upload_data( path=os.path.join(mnist_dir, "training"), key_prefix="integ-test-data/pytorch_mnist/training", ) tuning_job_name = unique_name_from_base("pytorch", max_length=32) tuner.fit({"training": training_data}, job_name=tuning_job_name) print("Started hyperparameter tuning job with name:" + tuning_job_name) time.sleep(15) tuner.wait() attached_tuner = HyperparameterTuner.attach( tuning_job_name, sagemaker_session=sagemaker_session) assert attached_tuner.early_stopping_type == "Auto" best_training_job = tuner.best_training_job() with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session): predictor = attached_tuner.deploy(1, "ml.c4.xlarge") data = np.zeros(shape=(1, 1, 28, 28), dtype=np.float32) predictor.predict(data) batch_size = 100 data = np.random.rand(batch_size, 1, 28, 28).astype(np.float32) output = predictor.predict(data) assert output.shape == (batch_size, 10)
def test_attach_tuning_job_with_estimator_from_image(sagemaker_session): job_details = copy.deepcopy(TUNING_JOB_DETAILS) job_details['TrainingJobDefinition']['AlgorithmSpecification']['TrainingImage'] = '1111.amazonaws.com/pca:1' sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job', return_value=job_details) tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session) assert isinstance(tuner.estimator, PCA)
def test_attach_with_no_specified_estimator(sagemaker_session): job_details = copy.deepcopy(TUNING_JOB_DETAILS) del job_details['TrainingJobDefinition']['StaticHyperParameters']['sagemaker_estimator_module'] del job_details['TrainingJobDefinition']['StaticHyperParameters']['sagemaker_estimator_class_name'] sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job', return_value=job_details) tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session) assert isinstance(tuner.estimator, Estimator)
def test_identical_dataset_and_algorithm_tuner(sagemaker_session): job_details = copy.deepcopy(TUNING_JOB_DETAILS) sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job', return_value=job_details) tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session) parent_tuner = tuner.identical_dataset_and_algorithm_tuner(additional_parents={"p1", "p2"}) assert parent_tuner.warm_start_config.type == WarmStartTypes.IDENTICAL_DATA_AND_ALGORITHM assert parent_tuner.warm_start_config.parents == {tuner.latest_tuning_job.name, "p1", "p2"}
def test_attach_tuning_pytorch(sagemaker_session): mnist_dir = os.path.join(DATA_DIR, 'pytorch_mnist') mnist_script = os.path.join(mnist_dir, 'mnist.py') estimator = PyTorch(entry_point=mnist_script, role='SageMakerRole', train_instance_count=1, py_version=PYTHON_VERSION, train_instance_type='ml.c4.xlarge', sagemaker_session=sagemaker_session) with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES): objective_metric_name = 'evaluation-accuracy' metric_definitions = [{ 'Name': 'evaluation-accuracy', 'Regex': r'Overall test accuracy: (\d+)' }] hyperparameter_ranges = {'batch-size': IntegerParameter(50, 100)} tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions, max_jobs=2, max_parallel_jobs=2, early_stopping_type='Auto') training_data = estimator.sagemaker_session.upload_data( path=os.path.join(mnist_dir, 'training'), key_prefix='integ-test-data/pytorch_mnist/training') tuning_job_name = unique_name_from_base('pytorch', max_length=32) tuner.fit({'training': training_data}, job_name=tuning_job_name) print('Started hyperparameter tuning job with name:' + tuning_job_name) time.sleep(15) tuner.wait() attached_tuner = HyperparameterTuner.attach( tuning_job_name, sagemaker_session=sagemaker_session) assert attached_tuner.early_stopping_type == 'Auto' best_training_job = tuner.best_training_job() with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session): predictor = attached_tuner.deploy(1, 'ml.c4.xlarge') data = np.zeros(shape=(1, 1, 28, 28), dtype=np.float32) predictor.predict(data) batch_size = 100 data = np.random.rand(batch_size, 1, 28, 28).astype(np.float32) output = predictor.predict(data) assert output.shape == (batch_size, 10)
def test_attach_tuning_job_with_estimator_from_hyperparameters_with_early_stopping(sagemaker_session): job_details = copy.deepcopy(TUNING_JOB_DETAILS) job_details['HyperParameterTuningJobConfig']['TrainingJobEarlyStoppingType'] = 'Auto' sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job', return_value=job_details) tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session) assert tuner.latest_tuning_job.name == JOB_NAME assert tuner.early_stopping_type == 'Auto' assert isinstance(tuner.estimator, PCA)
def test_transfer_learning_tuner(sagemaker_session): job_details = copy.deepcopy(TUNING_JOB_DETAILS) sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job', return_value=job_details) tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session) parent_tuner = tuner.transfer_learning_tuner(additional_parents={"p1", "p2"}) assert parent_tuner.warm_start_config.type == WarmStartTypes.TRANSFER_LEARNING assert parent_tuner.warm_start_config.parents == {tuner.latest_tuning_job.name, "p1", "p2"} assert parent_tuner.estimator == tuner.estimator
def test_attach_with_warm_start_config(sagemaker_session): warm_start_config = WarmStartConfig(warm_start_type=WarmStartTypes.TRANSFER_LEARNING, parents={"p1", "p2"}) job_details = copy.deepcopy(TUNING_JOB_DETAILS) job_details["WarmStartConfig"] = warm_start_config.to_input_req() sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job', return_value=job_details) tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session) assert tuner.warm_start_config.type == warm_start_config.type assert tuner.warm_start_config.parents == warm_start_config.parents
def _attach_tuner(sagemaker_session, tuning_job_name): print("Attaching hyperparameter tuning job {} to a new tuner instance". format(tuning_job_name)) return HyperparameterTuner.attach( tuning_job_name, sagemaker_session=sagemaker_session, estimator_cls={ ESTIMATOR_FM: "sagemaker.estimator.Estimator", ESTIMATOR_KNN: "sagemaker.estimator.Estimator", }, )
def test_attach_tuning_pytorch(sagemaker_session): mnist_dir = os.path.join(DATA_DIR, 'pytorch_mnist') mnist_script = os.path.join(mnist_dir, 'mnist.py') estimator = PyTorch(entry_point=mnist_script, role='SageMakerRole', train_instance_count=1, train_instance_type='ml.c4.xlarge', sagemaker_session=sagemaker_session) with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES): objective_metric_name = 'evaluation-accuracy' metric_definitions = [{'Name': 'evaluation-accuracy', 'Regex': 'Overall test accuracy: (\d+)'}] hyperparameter_ranges = {'batch-size': IntegerParameter(50, 100)} tuner = HyperparameterTuner(estimator, objective_metric_name, hyperparameter_ranges, metric_definitions, max_jobs=2, max_parallel_jobs=2) training_data = estimator.sagemaker_session.upload_data(path=os.path.join(mnist_dir, 'training'), key_prefix='integ-test-data/pytorch_mnist/training') tuner.fit({'training': training_data}) tuning_job_name = tuner.latest_tuning_job.name print('Started hyperparameter tuning job with name:' + tuning_job_name) time.sleep(15) tuner.wait() attached_tuner = HyperparameterTuner.attach(tuning_job_name, sagemaker_session=sagemaker_session) best_training_job = tuner.best_training_job() with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session): predictor = attached_tuner.deploy(1, 'ml.c4.xlarge') data = np.zeros(shape=(1, 1, 28, 28), dtype=np.float32) predictor.predict(data) batch_size = 100 data = np.random.rand(batch_size, 1, 28, 28).astype(np.float32) output = predictor.predict(data) assert output.shape == (batch_size, 10)
def test_attach_tuning_job_with_estimator_from_hyperparameters(sagemaker_session): job_details = copy.deepcopy(TUNING_JOB_DETAILS) sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(name='describe_tuning_job', return_value=job_details) tuner = HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session) assert tuner.latest_tuning_job.name == JOB_NAME assert tuner.objective_metric_name == OBJECTIVE_METRIC_NAME assert tuner.max_jobs == 1 assert tuner.max_parallel_jobs == 1 assert tuner.metric_definitions == METRIC_DEFINTIONS assert tuner.strategy == 'Bayesian' assert tuner.objective_type == 'Minimize' assert isinstance(tuner.estimator, PCA) assert tuner.estimator.role == ROLE assert tuner.estimator.train_instance_count == 1 assert tuner.estimator.train_max_run == 24 * 60 * 60 assert tuner.estimator.input_mode == 'File' assert tuner.estimator.output_path == BUCKET_NAME assert tuner.estimator.output_kms_key == '' assert '_tuning_objective_metric' not in tuner.estimator.hyperparameters() assert tuner.estimator.hyperparameters()['num_components'] == '1'
def test_attach_tuning_job_with_job_details(sagemaker_session): job_details = copy.deepcopy(TUNING_JOB_DETAILS) HyperparameterTuner.attach(JOB_NAME, sagemaker_session=sagemaker_session, job_details=job_details) sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job.assert_not_called
def test_tuning_lda(sagemaker_session, cpu_instance_type): with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES): data_path = os.path.join(DATA_DIR, "lda") data_filename = "nips-train_1.pbr" with open(os.path.join(data_path, data_filename), "rb") as f: all_records = read_records(f) # all records must be same feature_num = int( all_records[0].features["values"].float32_tensor.shape[0]) lda = LDA( role="SageMakerRole", instance_type=cpu_instance_type, num_topics=10, sagemaker_session=sagemaker_session, ) record_set = prepare_record_set_from_local_files( data_path, lda.data_location, len(all_records), feature_num, sagemaker_session) test_record_set = prepare_record_set_from_local_files( data_path, lda.data_location, len(all_records), feature_num, sagemaker_session) test_record_set.channel = "test" # specify which hp you want to optimize over hyperparameter_ranges = { "alpha0": ContinuousParameter(1, 10), "num_topics": IntegerParameter(1, 2), } objective_metric_name = "test:pwll" tuner = HyperparameterTuner( estimator=lda, objective_metric_name=objective_metric_name, hyperparameter_ranges=hyperparameter_ranges, objective_type="Maximize", max_jobs=2, max_parallel_jobs=2, early_stopping_type="Auto", ) tuning_job_name = unique_name_from_base("test-lda", max_length=32) print("Started hyperparameter tuning job with name:" + tuning_job_name) tuner.fit([record_set, test_record_set], mini_batch_size=1, job_name=tuning_job_name) attached_tuner = HyperparameterTuner.attach( tuning_job_name, sagemaker_session=sagemaker_session) assert attached_tuner.early_stopping_type == "Auto" assert attached_tuner.estimator.alpha0 == 1.0 assert attached_tuner.estimator.num_topics == 1 best_training_job = attached_tuner.best_training_job() with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session): predictor = tuner.deploy(1, cpu_instance_type) predict_input = np.random.rand(1, feature_num) result = predictor.predict(predict_input) assert len(result) == 1 for record in result: assert record.label["topic_mixture"] is not None