def test_init_sigma_fail_value_high(sagemaker_session): with pytest.raises(ValueError): LinearLearner(init_sigma=1, sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_async_linear_learner(sagemaker_session): training_job_name = "" endpoint_name = 'test-linear-learner-async-{}'.format( sagemaker_timestamp()) with timeout(minutes=5): data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz') pickle_args = {} if sys.version_info.major == 2 else { 'encoding': 'latin1' } # Load the data into memory as numpy arrays with gzip.open(data_path, 'rb') as f: train_set, _, _ = pickle.load(f, **pickle_args) train_set[1][:100] = 1 train_set[1][100:200] = 0 train_set = train_set[0], train_set[1].astype(np.dtype('float32')) ll = LinearLearner('SageMakerRole', 1, 'ml.c4.2xlarge', base_job_name='test-linear-learner', predictor_type='binary_classifier', sagemaker_session=sagemaker_session) ll.binary_classifier_model_selection_criteria = 'accuracy' ll.target_recall = 0.5 ll.target_precision = 0.5 ll.positive_example_weight_mult = 0.1 ll.epochs = 1 ll.use_bias = True ll.num_models = 1 ll.num_calibration_samples = 1 ll.init_method = 'uniform' ll.init_scale = 0.5 ll.init_sigma = 0.2 ll.init_bias = 5 ll.optimizer = 'adam' ll.loss = 'logistic' ll.wd = 0.5 ll.l1 = 0.5 ll.momentum = 0.5 ll.learning_rate = 0.1 ll.beta_1 = 0.1 ll.beta_2 = 0.1 ll.use_lr_scheduler = True ll.lr_scheduler_step = 2 ll.lr_scheduler_factor = 0.5 ll.lr_scheduler_minimum_lr = 0.1 ll.normalize_data = False ll.normalize_label = False ll.unbias_data = True ll.unbias_label = False ll.num_point_for_scaler = 10000 ll.margin = 1.0 ll.quantile = 0.5 ll.loss_insensitivity = 0.1 ll.huber_delta = 0.1 ll.early_stopping_tolerance = 0.0001 ll.early_stopping_patience = 3 ll.fit(ll.record_set(train_set[0][:200], train_set[1][:200]), wait=False) training_job_name = ll.latest_training_job.name print("Waiting to re-attach to the training job: %s" % training_job_name) time.sleep(20) with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, minutes=35): estimator = LinearLearner.attach(training_job_name=training_job_name, sagemaker_session=sagemaker_session) model = LinearLearnerModel(estimator.model_data, role='SageMakerRole', sagemaker_session=sagemaker_session) predictor = model.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name) result = predictor.predict(train_set[0][0:100]) assert len(result) == 100 for record in result: assert record.label["predicted_label"] is not None assert record.label["score"] is not None
def test_image(sagemaker_session): lr = LinearLearner(sagemaker_session=sagemaker_session, **ALL_REQ_ARGS) assert lr.train_image() == registry(REGION, 'linear-learner') + '/linear-learner:1'
def test_iterable_hyper_parameters_type(sagemaker_session, iterable_hyper_parameters, value): with pytest.raises(TypeError): test_params = ALL_REQ_ARGS.copy() test_params.update({iterable_hyper_parameters: value}) LinearLearner(sagemaker_session=sagemaker_session, **test_params)
def test_all_hyperparameters(sagemaker_session): lr = LinearLearner( sagemaker_session=sagemaker_session, binary_classifier_model_selection_criteria="accuracy", target_recall=0.5, target_precision=0.6, positive_example_weight_mult=0.1, epochs=1, use_bias=True, num_models=5, num_calibration_samples=6, init_method="uniform", init_scale=0.1, init_sigma=0.001, init_bias=0, optimizer="sgd", loss="logistic", wd=0.4, l1=0.04, momentum=0.1, learning_rate=0.001, beta_1=0.2, beta_2=0.03, bias_lr_mult=5.5, bias_wd_mult=6.6, use_lr_scheduler=False, lr_scheduler_step=2, lr_scheduler_factor=0.03, lr_scheduler_minimum_lr=0.001, normalize_data=False, normalize_label=True, unbias_data=True, unbias_label=False, num_point_for_scaler=3, margin=1.0, quantile=0.5, loss_insensitivity=0.1, huber_delta=0.1, early_stopping_patience=3, early_stopping_tolerance=0.001, num_classes=1, accuracy_top_k=3, f_beta=1.0, balance_multiclass_weights=False, **ALL_REQ_ARGS ) assert lr.hyperparameters() == dict( predictor_type="binary_classifier", binary_classifier_model_selection_criteria="accuracy", target_recall="0.5", target_precision="0.6", positive_example_weight_mult="0.1", epochs="1", use_bias="True", num_models="5", num_calibration_samples="6", init_method="uniform", init_scale="0.1", init_sigma="0.001", init_bias="0.0", optimizer="sgd", loss="logistic", wd="0.4", l1="0.04", momentum="0.1", learning_rate="0.001", beta_1="0.2", beta_2="0.03", bias_lr_mult="5.5", bias_wd_mult="6.6", use_lr_scheduler="False", lr_scheduler_step="2", lr_scheduler_factor="0.03", lr_scheduler_minimum_lr="0.001", normalize_data="False", normalize_label="True", unbias_data="True", unbias_label="False", num_point_for_scaler="3", margin="1.0", quantile="0.5", loss_insensitivity="0.1", huber_delta="0.1", early_stopping_patience="3", early_stopping_tolerance="0.001", num_classes="1", accuracy_top_k="3", f_beta="1.0", balance_multiclass_weights="False", )
def test_all_hyperparameters(sagemaker_session): lr = LinearLearner(sagemaker_session=sagemaker_session, binary_classifier_model_selection_criteria='accuracy', target_recall=0.5, target_precision=0.6, positive_example_weight_mult=0.1, epochs=1, use_bias=True, num_models=5, num_calibration_samples=6, init_method='uniform', init_scale=0.1, init_sigma=0.001, init_bias=0, optimizer='sgd', loss='logistic', wd=0.4, l1=0.04, momentum=0.1, learning_rate=0.001, beta_1=0.2, beta_2=0.03, bias_lr_mult=5.5, bias_wd_mult=6.6, use_lr_scheduler=False, lr_scheduler_step=2, lr_scheduler_factor=0.03, lr_scheduler_minimum_lr=0.001, normalize_data=False, normalize_label=True, unbias_data=True, unbias_label=False, num_point_for_scaler=3, margin=1.0, quantile=0.5, loss_insensitivity=0.1, huber_delta=0.1, early_stopping_patience=3, early_stopping_tolerance=0.001, num_classes=1, accuracy_top_k=3, f_beta=1.0, balance_multiclass_weights=False, **ALL_REQ_ARGS) assert lr.hyperparameters() == dict( predictor_type='binary_classifier', binary_classifier_model_selection_criteria='accuracy', target_recall='0.5', target_precision='0.6', positive_example_weight_mult='0.1', epochs='1', use_bias='True', num_models='5', num_calibration_samples='6', init_method='uniform', init_scale='0.1', init_sigma='0.001', init_bias='0.0', optimizer='sgd', loss='logistic', wd='0.4', l1='0.04', momentum='0.1', learning_rate='0.001', beta_1='0.2', beta_2='0.03', bias_lr_mult='5.5', bias_wd_mult='6.6', use_lr_scheduler='False', lr_scheduler_step='2', lr_scheduler_factor='0.03', lr_scheduler_minimum_lr='0.001', normalize_data='False', normalize_label='True', unbias_data='True', unbias_label='False', num_point_for_scaler='3', margin='1.0', quantile='0.5', loss_insensitivity='0.1', huber_delta='0.1', early_stopping_patience='3', early_stopping_tolerance='0.001', num_classes='1', accuracy_top_k='3', f_beta='1.0', balance_multiclass_weights='False', )
def test_linear_learner(sagemaker_session, cpu_instance_type, training_set): job_name = unique_name_from_base("linear-learner") with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES): training_set[1][:100] = 1 training_set[1][100:200] = 0 training_set = training_set[0], training_set[1].astype( np.dtype("float32")) ll = LinearLearner( "SageMakerRole", 1, cpu_instance_type, predictor_type="binary_classifier", sagemaker_session=sagemaker_session, ) ll.binary_classifier_model_selection_criteria = "accuracy" ll.target_recall = 0.5 ll.target_precision = 0.5 ll.positive_example_weight_mult = 0.1 ll.epochs = 1 ll.use_bias = True ll.num_models = 1 ll.num_calibration_samples = 1 ll.init_method = "uniform" ll.init_scale = 0.5 ll.init_sigma = 0.2 ll.init_bias = 5 ll.optimizer = "adam" ll.loss = "logistic" ll.wd = 0.5 ll.l1 = 0.5 ll.momentum = 0.5 ll.learning_rate = 0.1 ll.beta_1 = 0.1 ll.beta_2 = 0.1 ll.use_lr_scheduler = True ll.lr_scheduler_step = 2 ll.lr_scheduler_factor = 0.5 ll.lr_scheduler_minimum_lr = 0.1 ll.normalize_data = False ll.normalize_label = False ll.unbias_data = True ll.unbias_label = False ll.num_point_for_scaler = 10000 ll.margin = 1.0 ll.quantile = 0.5 ll.loss_insensitivity = 0.1 ll.huber_delta = 0.1 ll.early_stopping_tolerance = 0.0001 ll.early_stopping_patience = 3 ll.fit(ll.record_set(training_set[0][:200], training_set[1][:200]), job_name=job_name) with timeout_and_delete_endpoint_by_name(job_name, sagemaker_session): predictor = ll.deploy(1, cpu_instance_type, endpoint_name=job_name) result = predictor.predict(training_set[0][0:100]) assert len(result) == 100 for record in result: assert record.label["predicted_label"] is not None assert record.label["score"] is not None
def test_learning_rate_fail_value_high(sagemaker_session): with pytest.raises(ValueError): LinearLearner(learning_rate=1, sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_learning_rate_fail_type(sagemaker_session): with pytest.raises(ValueError): LinearLearner(learning_rate='blah', sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_momentum_fail_value_high(sagemaker_session): with pytest.raises(ValueError): LinearLearner(momentum=1, sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_momentum_fail_type(sagemaker_session): with pytest.raises(ValueError): LinearLearner(momentum='blah', sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_loss_fail(sagemaker_session): with pytest.raises(ValueError): LinearLearner(loss='other', sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_optimizer_fail(sagemaker_session): with pytest.raises(ValueError): LinearLearner(optimizer='other', sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_init_sigma_fail_type(sagemaker_session): with pytest.raises(ValueError): LinearLearner(init_sigma='blah', sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_lr_scheduler_minimum_lr_fail_type(sagemaker_session): with pytest.raises(ValueError): LinearLearner(lr_scheduler_minimum_lr='blah', sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_beta_1_fail_value_low(sagemaker_session): with pytest.raises(ValueError): LinearLearner(beta_1=0, sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_num_point_for_scaler_fail_type(sagemaker_session): with pytest.raises(ValueError): LinearLearner(num_point_for_scaler='other', sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_beta_2_fail_value_high(sagemaker_session): with pytest.raises(ValueError): LinearLearner(beta_2=1, sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_image(sagemaker_session): lr = LinearLearner(sagemaker_session=sagemaker_session, **ALL_REQ_ARGS) assert image_uris.retrieve("linear-learner", REGION) == lr.training_image_uri()
def test_beta_2_fail_type(sagemaker_session): with pytest.raises(ValueError): LinearLearner(beta_2='blah', sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_num_classes_can_be_string_for_multiclass_classifier(sagemaker_session): test_params = ALL_REQ_ARGS.copy() test_params["predictor_type"] = "multiclass_classifier" test_params["num_classes"] = "3" LinearLearner(sagemaker_session=sagemaker_session, **test_params)
def test_bias_wd_mult_fail_value_low(sagemaker_session): with pytest.raises(ValueError): LinearLearner(bias_wd_mult=0, sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_optional_hyper_parameters_value(sagemaker_session, optional_hyper_parameters, value): with pytest.raises(ValueError): test_params = ALL_REQ_ARGS.copy() test_params.update({optional_hyper_parameters: value}) LinearLearner(sagemaker_session=sagemaker_session, **test_params)
def test_bias_wd_mult_fail_type(sagemaker_session): with pytest.raises(ValueError): LinearLearner(bias_wd_mult='blah', sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_linear_learner(): with timeout(minutes=15): sagemaker_session = sagemaker.Session(boto_session=boto3.Session( region_name=REGION)) data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz') pickle_args = {} if sys.version_info.major == 2 else { 'encoding': 'latin1' } # Load the data into memory as numpy arrays with gzip.open(data_path, 'rb') as f: train_set, _, _ = pickle.load(f, **pickle_args) train_set[1][:100] = 1 train_set[1][100:200] = 0 train_set = train_set[0], train_set[1].astype(np.dtype('float32')) ll = LinearLearner('SageMakerRole', 1, 'ml.c4.2xlarge', base_job_name='test-linear-learner', sagemaker_session=sagemaker_session) ll.binary_classifier_model_selection_criteria = 'accuracy' ll.target_recall = 0.5 ll.target_precision = 0.5 ll.positive_example_weight_mult = 0.1 ll.epochs = 1 ll.predictor_type = 'binary_classifier' ll.use_bias = True ll.num_models = 1 ll.num_calibration_samples = 1 ll.init_method = 'uniform' ll.init_scale = 0.5 ll.init_sigma = 0.2 ll.init_bias = 5 ll.optimizer = 'adam' ll.loss = 'logistic' ll.wd = 0.5 ll.l1 = 0.5 ll.momentum = 0.5 ll.learning_rate = 0.1 ll.beta_1 = 0.1 ll.beta_2 = 0.1 ll.use_lr_scheduler = True ll.lr_scheduler_step = 2 ll.lr_scheduler_factor = 0.5 ll.lr_scheduler_minimum_lr = 0.1 ll.normalize_data = False ll.normalize_label = False ll.unbias_data = True ll.unbias_label = False ll.num_point_for_scaler = 10000 ll.fit(ll.record_set(train_set[0][:200], train_set[1][:200])) endpoint_name = name_from_base('linear-learner') with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session, minutes=20): predictor = ll.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name) result = predictor.predict(train_set[0][0:100]) assert len(result) == 100 for record in result: assert record.label["predicted_label"] is not None assert record.label["score"] is not None
def test_lr_scheduler_step_fail_type(sagemaker_session): with pytest.raises(ValueError): LinearLearner(lr_scheduler_step='other', sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_linear_learner(sagemaker_session): job_name = unique_name_from_base("linear-learner") with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES): data_path = os.path.join(DATA_DIR, "one_p_mnist", "mnist.pkl.gz") pickle_args = {} if sys.version_info.major == 2 else { "encoding": "latin1" } # Load the data into memory as numpy arrays with gzip.open(data_path, "rb") as f: train_set, _, _ = pickle.load(f, **pickle_args) train_set[1][:100] = 1 train_set[1][100:200] = 0 train_set = train_set[0], train_set[1].astype(np.dtype("float32")) ll = LinearLearner( "SageMakerRole", 1, "ml.c4.2xlarge", predictor_type="binary_classifier", sagemaker_session=sagemaker_session, ) ll.binary_classifier_model_selection_criteria = "accuracy" ll.target_recall = 0.5 ll.target_precision = 0.5 ll.positive_example_weight_mult = 0.1 ll.epochs = 1 ll.use_bias = True ll.num_models = 1 ll.num_calibration_samples = 1 ll.init_method = "uniform" ll.init_scale = 0.5 ll.init_sigma = 0.2 ll.init_bias = 5 ll.optimizer = "adam" ll.loss = "logistic" ll.wd = 0.5 ll.l1 = 0.5 ll.momentum = 0.5 ll.learning_rate = 0.1 ll.beta_1 = 0.1 ll.beta_2 = 0.1 ll.use_lr_scheduler = True ll.lr_scheduler_step = 2 ll.lr_scheduler_factor = 0.5 ll.lr_scheduler_minimum_lr = 0.1 ll.normalize_data = False ll.normalize_label = False ll.unbias_data = True ll.unbias_label = False ll.num_point_for_scaler = 10000 ll.margin = 1.0 ll.quantile = 0.5 ll.loss_insensitivity = 0.1 ll.huber_delta = 0.1 ll.early_stopping_tolerance = 0.0001 ll.early_stopping_patience = 3 ll.fit(ll.record_set(train_set[0][:200], train_set[1][:200]), job_name=job_name) with timeout_and_delete_endpoint_by_name(job_name, sagemaker_session): predictor = ll.deploy(1, "ml.c4.xlarge", endpoint_name=job_name) result = predictor.predict(train_set[0][0:100]) assert len(result) == 100 for record in result: assert record.label["predicted_label"] is not None assert record.label["score"] is not None
def test_lr_scheduler_factor_fail_value_high(sagemaker_session): with pytest.raises(ValueError): LinearLearner(lr_scheduler_factor=1, sagemaker_session=sagemaker_session, **REQ_ARGS)
def test_required_hyper_parameters_value(sagemaker_session, required_hyper_parameters, value): with pytest.raises(ValueError): test_params = ALL_REQ_ARGS.copy() test_params[required_hyper_parameters] = value LinearLearner(sagemaker_session=sagemaker_session, **test_params)
def test_init_scale_fail_value_low(sagemaker_session): with pytest.raises(ValueError): LinearLearner(init_scale=1.01, sagemaker_session=sagemaker_session, **REQ_ARGS)