def test_partial_fit_single_epoch(self): """Test that partial_fit trains for a single epoch, independently of what epoch value is passed to the constructor. """ data = fetch_california_housing() X, y = data.data[:100], data.target[:100] epochs = 9 partial_fit_iter = 4 estimator = KerasRegressor( model=dynamic_regressor, model__hidden_layer_sizes=[ 100, ], epochs=epochs, ) # Check that each partial_fit call trains for 1 epoch for k in range(1, partial_fit_iter): estimator = estimator.partial_fit(X, y) assert len(estimator.history_["loss"]) == k # Check that fit calls still train for the number of # epochs specified in the constructor estimator = estimator.fit(X, y) assert len(estimator.history_["loss"]) == epochs
def test_history(self): """Test that history_'s keys are strings and values are lists.""" data = fetch_california_housing() X, y = data.data[:100], data.target[:100] estimator = KerasRegressor( model=dynamic_regressor, model__hidden_layer_sizes=[] ) estimator.partial_fit(X, y) assert isinstance(estimator.history_, dict) assert all(isinstance(k, str) for k in estimator.history_.keys()) assert all(isinstance(v, list) for v in estimator.history_.values())
def test_no_attributes_set_init_no_args(): """Tests that models with no build arguments set all parameters in a single __init__ """ def build_fn(): model = Sequential() model.add(layers.Dense(1, input_dim=1, activation="relu")) model.add(layers.Dense(1)) model.compile(loss="mse") return model estimator = KerasRegressor(model=build_fn) check_no_attributes_set_in_init(estimator.__name__, estimator) estimator.fit([[1]], [1])
def test_current_epoch_property(self, warm_start): """Test the public current_epoch property that tracks the overall training epochs. The warm_start parameter should have NO impact on this behavior. """ data = load_boston() X, y = data.data[:100], data.target[:100] epochs = 2 partial_fit_iter = 4 estimator = KerasRegressor( model=dynamic_regressor, loss=KerasRegressor.r_squared, model__hidden_layer_sizes=[ 100, ], epochs=epochs, warm_start=warm_start, ) # Check that each partial_fit call trains for 1 epoch for k in range(1, partial_fit_iter): estimator.partial_fit(X, y) assert estimator.current_epoch == k # Check that fit calls still train for the number of # epochs specified in the constructor estimator.fit(X, y) assert estimator.current_epoch == epochs # partial_fit is able to resume from a non-zero epoch estimator.partial_fit(X, y) assert estimator.current_epoch == epochs + 1
def test_partial_fit_shorthand_metric_name(self): """Test that metrics get stored in the `history_` attribute by their long name (and not shorthand) even if the user compiles their model with a shorthand name. """ est = KerasRegressor( model=force_compile_shorthand, model__hidden_layer_sizes=(100,), metrics=["mae"], # shorthand ) X, y = fetch_california_housing(return_X_y=True) X = X[:100] y = y[:100] est.fit(X, y) assert "mae" not in est.history_ and "mean_absolute_error" in est.history_
def test_sample_weights_score(): """Checks that the `sample_weight` parameter when passed to `score` has the intended effect. """ # build estimator estimator = KerasRegressor( model=dynamic_regressor, model__hidden_layer_sizes=(100, ), epochs=10, random_state=0, ) estimator1 = clone(estimator) estimator2 = clone(estimator) # we create 20 points X = np.array([1] * 10000).reshape(-1, 1) y = [1] * 5000 + [-1] * 5000 # train estimator1.fit(X, y) estimator2.fit(X, y) # heavily weight towards y=1 points bad_sw = [0.999] * 5000 + [0.001] * 5000 # score with weights, estimator2 should # score higher since the weights "unbalance" score1 = estimator1.score(X, y, sample_weight=bad_sw) score2 = estimator2.score(X, y) assert score2 > score1
def test_no_loss(loss, compile): def get_model(compile, meta, compile_kwargs): inp = Input(shape=(meta["n_features_in_"], )) hidden = Dense(10, activation="relu")(inp) out = [ Dense(1, activation="sigmoid", name=f"out{i+1}")(hidden) for i in range(meta["n_outputs_"]) ] model = Model(inp, out) if compile: model.compile(**compile_kwargs) return model est = KerasRegressor(model=get_model, loss=loss, compile=compile) with pytest.raises(ValueError, match="must provide a loss function"): est.fit([[0], [1]], [0, 1])
def keras_backend_r2(y_true, y_pred): """Wrap Keras operations to numpy.""" y_true = convert_to_tensor(y_true) y_pred = convert_to_tensor(y_pred) return KerasRegressor.root_mean_squared_error( y_true, y_pred ).numpy()
def test_custom_loss_function(): """Test that a custom loss function can be serialized.""" estimator = KerasRegressor( model=dynamic_regressor, loss=CustomLoss(), model__hidden_layer_sizes=(100, ), ) check_pickle(estimator, fetch_california_housing)
def test_run_eagerly(): """Test that models compiled with run_eagerly can be serialized.""" estimator = KerasRegressor( model=dynamic_regressor, run_eagerly=True, model__hidden_layer_sizes=(100, ), ) check_pickle(estimator, fetch_california_housing)
def test_pickle_with_callbacks(): """Test that models with callbacks (which hold a refence to the Keras model itself) are picklable.""" clf = KerasRegressor(model=get_reg, loss="mse", callbacks=[keras.callbacks.Callback()]) # Fit and roundtrip validating only that there are no errors clf.fit([[1]], [1]) clf = pickle.loads(pickle.dumps(clf)) clf.predict([[1]]) clf.partial_fit([[1]], [1])
def test_partial_fit_pickle(optim): """ This test is implemented to make sure model pickling does not affect training. (this is essentially what Dask-ML does for search) """ X, y = make_regression(n_features=8, n_samples=100) m1 = KerasRegressor(model=dynamic_regressor, optimizer=optim, random_state=42, hidden_layer_sizes=[]) m2 = clone(m1) # Ensure we can roundtrip before training m2 = _reload(m2) # Make sure start from same model m1.partial_fit(X, y) m2.partial_fit(X, y) assert _weights_close(m1, m2) # Train; make sure pickling doesn't affect it for k in range(4): m1.partial_fit(X, y) m2 = _reload(m2, epoch=k + 1).partial_fit(X, y) # Make sure the same model is produced assert _weights_close(m1, m2) # Make sure predictions are the same assert np.allclose(m1.predict(X), m2.predict(X))
def test_no_optimizer(compile): def get_model(compile, meta, compile_kwargs): inp = Input(shape=(meta["n_features_in_"],)) hidden = Dense(10, activation="relu")(inp) out = [ Dense(1, activation="sigmoid", name=f"out{i+1}")(hidden) for i in range(meta["n_outputs_"]) ] model = Model(inp, out) if compile: model.compile(**compile_kwargs) return model est = KerasRegressor(model=get_model, loss="mse", compile=compile, optimizer=None,) with pytest.raises( ValueError, match="Could not interpret optimizer identifier" # Keras error ): est.fit([[0], [1]], [0, 1])
def test_kerasregressor_r2_as_metric(): """Test custom R^2 implementation against scikit-learn's.""" est = KerasRegressor(dynamic_regressor, metrics=[KerasRegressor.r_squared], epochs=10, random_state=0) y = np.random.randint(low=0, high=2, size=(1000, )) X = y.reshape((-1, 1)) est.fit(X, y) current_score = est.score(X, y) last_hist = est.history_["r_squared"][-1] np.testing.assert_almost_equal(current_score, last_hist, decimal=3) current_eval = est.model_.evaluate(X, y, return_dict=True)["r_squared"] np.testing.assert_almost_equal(current_score, current_eval, decimal=3)
def test_run_eagerly(): """Test that models compiled with run_eagerly can be serialized. """ estimator = KerasRegressor( model=dynamic_regressor, run_eagerly=True, loss=KerasRegressor.r_squared, model__hidden_layer_sizes=(100, ), ) check_pickle(estimator, load_boston)
def test_partial_fit_history_metric_names(self): data = fetch_california_housing() X, y = data.data[:100], data.target[:100] estimator = KerasRegressor( model=dynamic_regressor, model__hidden_layer_sizes=[ 100, ], metrics=["mse", CustomMetric(name="custom_metric")], ) estimator.partial_fit(X, y) # Make custom metric names are preserved # and shorthand metric names are saved by their full name for _ in range(2): estimator = pickle.loads(pickle.dumps(estimator)) estimator = estimator.partial_fit(X, y) assert set(estimator.history_.keys()) == { "loss", "mean_squared_error", "custom_metric", }
def test_kerasregressor_r2_as_metric_in_model(): """Test custom R^2 implementation as part of a model""" epochs = 25 est = KerasRegressor( dynamic_regressor, metrics=[KerasRegressor.r_squared], epochs=epochs, random_state=42, ) y = np.random.uniform(size=(1000,)) X = y.reshape((-1, 1)) est.fit(X, y) scores = np.array(est.history_["r_squared"]) # basic sanity check assert np.all(scores <= 1) and len(scores) == epochs, scores # rough estimate of expected end result given the random seed assert scores[-1] > 0.9, scores
def test_partial_fit_history_len(self): # history_ records the history from this partial_fit call # Make sure for each call to partial_fit a single entry # into the history is added # As per https://github.com/keras-team/keras/issues/1766, # there is no direct measure of epochs data = fetch_california_housing() X, y = data.data[:100], data.target[:100] estimator = KerasRegressor( model=dynamic_regressor, metrics="mean_squared_error", model__hidden_layer_sizes=[ 100, ], ) for k in range(10): estimator = estimator.partial_fit(X, y) assert len(estimator.history_["loss"]) == k + 1 assert set(estimator.history_.keys()) == { "loss", "mean_squared_error", }
def test_current_epoch_property(self, warm_start, epochs_prefix): """Test the public current_epoch property that tracks the overall training epochs. The warm_start parameter should have NO impact on this behavior. The prefix should NOT have any impact on behavior. It is tested because the epochs param has special handling within param routing. """ data = load_boston() X, y = data.data[:10], data.target[:10] epochs = 2 partial_fit_iter = 3 estimator = KerasRegressor( model=dynamic_regressor, loss=KerasRegressor.r_squared, model__hidden_layer_sizes=[], warm_start=warm_start, ) estimator.set_params(**{epochs_prefix + "epochs": epochs}) # Check that each partial_fit call trains for 1 epoch for k in range(1, partial_fit_iter): estimator.partial_fit(X, y) assert estimator.current_epoch == k # Check that fit calls still train for the number of # epochs specified in the constructor estimator.fit(X, y) assert estimator.current_epoch == epochs # partial_fit is able to resume from a non-zero epoch estimator.partial_fit(X, y) assert estimator.current_epoch == epochs + 1
def test_target_shape_changes_incremental_fit_reg(): X = np.array([[1, 2], [2, 3]]) y = np.array([1, 3]).reshape(-1, 1) est = KerasRegressor(model=dynamic_regressor, hidden_layer_sizes=(100,)) est.fit(X, y) with pytest.raises( ValueError, match="Detected `y` to have ", ): est.partial_fit(X, np.column_stack([y, y]))
def test_partial_fit(self): data = fetch_california_housing() X, y = data.data[:100], data.target[:100] estimator = KerasRegressor( model=dynamic_regressor, model__hidden_layer_sizes=[ 100, ], ) estimator.partial_fit(X, y) # Make sure loss history is incremented assert len(estimator.history_["loss"]) == 1 estimator.partial_fit(X, y) assert len(estimator.history_["loss"]) == 2 # Make sure new model not created model = estimator.model_ estimator.partial_fit(X, y) assert estimator.model_ is model, "Model memory address should remain constant"
def test_compile_model_from_params(): """Tests that if build_fn returns an un-compiled model, the __init__ parameters will be used to compile it and that if build_fn returns a compiled model it is not re-compiled. """ # Load data data = load_boston() X, y = data.data[:100], data.target[:100] losses = ("mean_squared_error", "mean_absolute_error") # build_fn that does not compile def build_fn(compile_with_loss=None): model = Sequential() model.add(keras.layers.Dense(X.shape[1], input_shape=(X.shape[1], ))) model.add(keras.layers.Activation("relu")) model.add(keras.layers.Dense(1)) model.add(keras.layers.Activation("linear")) if compile_with_loss: model.compile(loss=compile_with_loss) return model for loss in losses: estimator = KerasRegressor( model=build_fn, loss=loss, # compile_with_loss=None returns an un-compiled model compile_with_loss=None, ) estimator.fit(X, y) assert estimator.model_.loss.__name__ == loss for myloss in losses: estimator = KerasRegressor( model=build_fn, loss="binary_crossentropy", # compile_with_loss != None overrides loss compile_with_loss=myloss, ) estimator.fit(X, y) assert estimator.model_.loss == myloss
def continuous(): # use ints so that we get measurable scores when castint to uint8 y = np.random.randint(low=0, high=2, size=(1000,)) X = y.reshape(-1, 1) sklearn_est = MLPRegressor(**mlp_kwargs) scikeras_est = KerasRegressor(dynamic_regressor, **scikeras_kwargs) for dtype in ("float32", "float64", "int64", "int32", "uint8", "uint16"): y_ = y.astype(dtype) yield TestParams( sklearn_est=sklearn_est, scikeras_est=scikeras_est, X=X, y=y_, X_expected_dtype_keras=X.dtype, y_expected_dtype_keras=dtype, min_score=0.99, scorer=r2_score, )
def test_X_shape_change(): """Tests that a ValueError is raised if the input changes shape in subsequent partial fit calls. """ estimator = KerasRegressor( model=dynamic_regressor, hidden_layer_sizes=(100, ), ) X = np.array([[1, 2], [3, 4]]).reshape(2, 2, 1) y = np.array([[0, 1, 0], [1, 0, 0]]) estimator.fit(X=X, y=y) with pytest.raises(ValueError, match="dimensions in X"): # Calling with a different number of dimensions for X raises an error estimator.partial_fit(X=X.reshape(2, 2), y=y)
def test_batch_size_all_fit(length, prefix, base): kw = prefix + base y = np.random.random((length, )) X = y.reshape((-1, 1)) est = KerasRegressor(dynamic_regressor, hidden_layer_sizes=[], **{kw: -1}) est.initialize(X, y) fit_orig = est.model_.fit def check_batch_size(**kwargs): assert kwargs[base] == X.shape[0] return fit_orig(**kwargs) with mock.patch.object(est.model_, "fit", new=check_batch_size): est.fit(X, y)
def test_shape_change_error(): """Tests that a ValueError is raised if the input changes shape in subsequent partial fit calls. """ estimator = KerasRegressor( model=dynamic_regressor, loss=KerasRegressor.r_squared, hidden_layer_sizes=(100, ), ) X = np.array([[1, 2], [3, 4]]) y = np.array([[0, 1, 0], [1, 0, 0]]) estimator.fit(X=X, y=y) with pytest.raises(ValueError, match=r"but this [\w\d]+ is expecting "): # Calling with a different shape for X raises an error estimator.partial_fit(X=X[:, :1], y=y)
def test_partial_fit_pickle(optim): """ This test is implemented to make sure model pickling does not affect training, which is (essentially) what Dask-ML does for a model selection search. This test is simple for functional optimizers (like SGD without momentum), and tricky for stateful transforms (SGD w/ momentum, Adam, Adagrad, etc). For more detail, see https://github.com/adriangb/scikeras/pull/126 and links within """ X, y = make_regression(n_features=8, n_samples=100) m1 = KerasRegressor(model=dynamic_regressor, optimizer=optim, random_state=42, hidden_layer_sizes=[]) m2 = clone(m1) # Ensure we can roundtrip before training m2 = _reload(m2) # Make sure start from same model m1.partial_fit(X, y) m2.partial_fit(X, y) assert _weights_close(m1, m2) # Train; make sure pickling doesn't affect it for k in range(4): m1.partial_fit(X, y) m2 = _reload(m2, epoch=k + 1).partial_fit(X, y) # Make sure the same model is produced assert _weights_close(m1, m2) # Make sure predictions are the same assert np.allclose(m1.predict(X), m2.predict(X))
assert y_out_scikeras.shape == y_out_sklearn.shape # Check dtype # By default, KerasRegressor (or rather it's default target_encoder) # always returns tf.keras.backend.floatx(). This is similar to sklearn, which always # returns float64, except that we avoid a pointless conversion from # float32 -> float64 that would just be adding noise if TF is using float32 # internally (which is usually the case). assert y_out_scikeras.dtype.name == tf.keras.backend.floatx() scikeras_score = test_data.scorer(y_test, y_out_scikeras) assert scikeras_score >= test_data.min_score @pytest.mark.parametrize( "est", ( KerasRegressor(dynamic_regressor, model__hidden_layer_sizes=[]), KerasClassifier(dynamic_classifier, model__hidden_layer_sizes=[]), ), ) @pytest.mark.parametrize( "X_dtype", ("float32", "float64", "int64", "int32", "uint8", "uint16", "object") ) def test_input_dtype_conversion(X_dtype, est): """Tests that using the default transformers in SciKeras, `X` is not converted/modified unless it is of dtype object. This mimics the behavior of sklearn estimators, which try to cast object -> numeric. """ y = np.arange(0, 10, 1, int) X = np.random.uniform(size=(y.shape[0], 2)).astype(X_dtype) est.fit(X, y) # generate model_
# slightly if X is shuffled. # This is only required for this tests and is not really # applicable to real world datasets batch_size = 1000 @parametrize_with_checks( estimators=[ MultiOutputClassifier( model=dynamic_classifier, batch_size=batch_size, model__hidden_layer_sizes=[], ), KerasRegressor( model=dynamic_regressor, batch_size=batch_size, model__hidden_layer_sizes=[], ), ], ) def test_fully_compliant_estimators_low_precision(estimator, check): """Checks that can be passed with sklearn's default tolerances and in a single epoch. """ check_name = check.func.__name__ if check_name in higher_precision: pytest.skip( "This test is run as part of test_fully_compliant_estimators_high_precision." ) check(estimator)
class TestRandomState: @pytest.mark.parametrize( "random_state", [0, 123, np.random.RandomState(0)], ) @pytest.mark.parametrize( "estimator", [ KerasRegressor( model=dynamic_regressor, loss=KerasRegressor.r_squared, model__hidden_layer_sizes=(100, ), ), KerasClassifier(model=dynamic_classifier, model__hidden_layer_sizes=(100, )), ], ) def test_random_states(self, random_state, estimator): """Tests that the random_state parameter correctly engages deterministric training and prediction. """ X, y = make_classification() # With seed estimator.set_params(random_state=random_state) estimator.fit(X, y) y1 = estimator.predict(X) estimator.fit(X, y) y2 = estimator.predict(X) assert np.allclose(y1, y2) if isinstance(estimator, KerasRegressor): # Without seed, regressors should NOT # give the same results # Classifiers _may_ give the same classes estimator.set_params(random_state=None) estimator.fit(X, y) y1 = estimator.predict(X) estimator.fit(X, y) y2 = estimator.predict(X) assert not np.allclose(y1, y2) @pytest.mark.parametrize( "estimator", [ KerasRegressor( model=dynamic_regressor, loss=KerasRegressor.r_squared, model__hidden_layer_sizes=(100, ), ), KerasClassifier(model=dynamic_classifier, model__hidden_layer_sizes=(100, )), ], ) @pytest.mark.parametrize("pyhash", [None, "0", "1"]) @pytest.mark.parametrize("gpu", [None, "0", "1"]) def test_random_states_env_vars(self, estimator, pyhash, gpu): """Tests that the random state context management correctly handles TF related env variables. """ X, y = make_classification() if "random_state" in estimator.get_params(): estimator.set_params(random_state=None) estimator1 = clone(estimator) estimator2 = clone(estimator) if "random_state" in estimator1.get_params(): estimator1.set_params(random_state=0) if "random_state" in estimator2.get_params(): estimator2.set_params(random_state=0) if gpu is not None: os.environ["TF_DETERMINISTIC_OPS"] = gpu else: if os.environ.get("TF_DETERMINISTIC_OPS"): os.environ.pop("TF_DETERMINISTIC_OPS") if pyhash is not None: os.environ["PYTHONHASHSEED"] = pyhash else: if os.environ.get("PYTHONHASHSEED"): os.environ.pop("PYTHONHASHSEED") estimator1.fit(X, y) estimator2.fit(X, y) if gpu is not None: assert os.environ["TF_DETERMINISTIC_OPS"] == gpu else: assert "TF_DETERMINISTIC_OPS" not in os.environ if pyhash is not None: assert os.environ["PYTHONHASHSEED"] == pyhash else: assert "PYTHONHASHSEED" not in os.environ y1 = estimator1.predict(X) y2 = estimator2.predict(X) assert np.allclose(y1, y2) if gpu is not None: assert os.environ["TF_DETERMINISTIC_OPS"] == gpu else: assert "TF_DETERMINISTIC_OPS" not in os.environ if pyhash is not None: assert os.environ["PYTHONHASHSEED"] == pyhash else: assert "PYTHONHASHSEED" not in os.environ