def test_current_epoch_property(self, warm_start):
"""Test the public current_epoch property
that tracks the overall training epochs.
The warm_start parameter should have
NO impact on this behavior.
"""
data = load_boston()
X, y = data.data[:100], data.target[:100]
epochs = 2
partial_fit_iter = 4
estimator = KerasRegressor(
model=dynamic_regressor,
loss=KerasRegressor.r_squared,
model__hidden_layer_sizes=[
100,
],
epochs=epochs,
warm_start=warm_start,
)
# Check that each partial_fit call trains for 1 epoch
for k in range(1, partial_fit_iter):
estimator.partial_fit(X, y)
assert estimator.current_epoch == k
# Check that fit calls still train for the number of
# epochs specified in the constructor
estimator.fit(X, y)
assert estimator.current_epoch == epochs
# partial_fit is able to resume from a non-zero epoch
estimator.partial_fit(X, y)
assert estimator.current_epoch == epochs + 1
def test_pickle_with_callbacks():
"""Test that models with callbacks (which hold a refence to the Keras model itself) are picklable."""
clf = KerasRegressor(model=get_reg,
loss="mse",
callbacks=[keras.callbacks.Callback()])
# Fit and roundtrip validating only that there are no errors
clf.fit([[1]], [1])
clf = pickle.loads(pickle.dumps(clf))
clf.predict([[1]])
clf.partial_fit([[1]], [1])
def test_target_shape_changes_incremental_fit_reg():
X = np.array([[1, 2], [2, 3]])
y = np.array([1, 3]).reshape(-1, 1)
est = KerasRegressor(model=dynamic_regressor, hidden_layer_sizes=(100,))
est.fit(X, y)
with pytest.raises(
ValueError, match="Detected `y` to have ",
):
est.partial_fit(X, np.column_stack([y, y]))
def test_no_attributes_set_init_no_args():
"""Tests that models with no build arguments
set all parameters in a single __init__
"""
def build_fn():
model = Sequential()
model.add(layers.Dense(1, input_dim=1, activation="relu"))
model.add(layers.Dense(1))
model.compile(loss="mse")
return model
estimator = KerasRegressor(model=build_fn)
check_no_attributes_set_in_init(estimator.__name__, estimator)
estimator.fit([[1]], [1])
def test_partial_fit_shorthand_metric_name(self):
"""Test that metrics get stored in the `history_` attribute
by their long name (and not shorthand) even if the user
compiles their model with a shorthand name.
"""
est = KerasRegressor(
model=force_compile_shorthand,
model__hidden_layer_sizes=(100,),
metrics=["mae"], # shorthand
)
X, y = fetch_california_housing(return_X_y=True)
X = X[:100]
y = y[:100]
est.fit(X, y)
assert "mae" not in est.history_ and "mean_absolute_error" in est.history_
def test_partial_fit_single_epoch(self):
"""Test that partial_fit trains for a single epoch,
independently of what epoch value is passed to the constructor.
"""
data = fetch_california_housing()
X, y = data.data[:100], data.target[:100]
epochs = 9
partial_fit_iter = 4
estimator = KerasRegressor(
model=dynamic_regressor,
model__hidden_layer_sizes=[
100,
],
epochs=epochs,
)
# Check that each partial_fit call trains for 1 epoch
for k in range(1, partial_fit_iter):
estimator = estimator.partial_fit(X, y)
assert len(estimator.history_["loss"]) == k
# Check that fit calls still train for the number of
# epochs specified in the constructor
estimator = estimator.fit(X, y)
assert len(estimator.history_["loss"]) == epochs
def test_no_loss(loss, compile):
def get_model(compile, meta, compile_kwargs):
inp = Input(shape=(meta["n_features_in_"], ))
hidden = Dense(10, activation="relu")(inp)
out = [
Dense(1, activation="sigmoid", name=f"out{i+1}")(hidden)
for i in range(meta["n_outputs_"])
]
model = Model(inp, out)
if compile:
model.compile(**compile_kwargs)
return model
est = KerasRegressor(model=get_model, loss=loss, compile=compile)
with pytest.raises(ValueError, match="must provide a loss function"):
est.fit([[0], [1]], [0, 1])
def test_X_shape_change():
"""Tests that a ValueError is raised if the input
changes shape in subsequent partial fit calls.
"""
estimator = KerasRegressor(
model=dynamic_regressor,
hidden_layer_sizes=(100, ),
)
X = np.array([[1, 2], [3, 4]]).reshape(2, 2, 1)
y = np.array([[0, 1, 0], [1, 0, 0]])
estimator.fit(X=X, y=y)
with pytest.raises(ValueError, match="dimensions in X"):
# Calling with a different number of dimensions for X raises an error
estimator.partial_fit(X=X.reshape(2, 2), y=y)
def test_kerasregressor_r2_as_metric():
"""Test custom R^2 implementation against scikit-learn's."""
est = KerasRegressor(dynamic_regressor,
metrics=[KerasRegressor.r_squared],
epochs=10,
random_state=0)
y = np.random.randint(low=0, high=2, size=(1000, ))
X = y.reshape((-1, 1))
est.fit(X, y)
current_score = est.score(X, y)
last_hist = est.history_["r_squared"][-1]
np.testing.assert_almost_equal(current_score, last_hist, decimal=3)
current_eval = est.model_.evaluate(X, y, return_dict=True)["r_squared"]
np.testing.assert_almost_equal(current_score, current_eval, decimal=3)
def test_shape_change_error():
"""Tests that a ValueError is raised if the input
changes shape in subsequent partial fit calls.
"""
estimator = KerasRegressor(
model=dynamic_regressor,
loss=KerasRegressor.r_squared,
hidden_layer_sizes=(100, ),
)
X = np.array([[1, 2], [3, 4]])
y = np.array([[0, 1, 0], [1, 0, 0]])
estimator.fit(X=X, y=y)
with pytest.raises(ValueError, match=r"but this [\w\d]+ is expecting "):
# Calling with a different shape for X raises an error
estimator.partial_fit(X=X[:, :1], y=y)
def test_no_optimizer(compile):
def get_model(compile, meta, compile_kwargs):
inp = Input(shape=(meta["n_features_in_"],))
hidden = Dense(10, activation="relu")(inp)
out = [
Dense(1, activation="sigmoid", name=f"out{i+1}")(hidden)
for i in range(meta["n_outputs_"])
]
model = Model(inp, out)
if compile:
model.compile(**compile_kwargs)
return model
est = KerasRegressor(model=get_model, loss="mse", compile=compile, optimizer=None,)
with pytest.raises(
ValueError, match="Could not interpret optimizer identifier" # Keras error
):
est.fit([[0], [1]], [0, 1])
def test_batch_size_all_fit(length, prefix, base):
kw = prefix + base
y = np.random.random((length, ))
X = y.reshape((-1, 1))
est = KerasRegressor(dynamic_regressor, hidden_layer_sizes=[], **{kw: -1})
est.initialize(X, y)
fit_orig = est.model_.fit
def check_batch_size(**kwargs):
assert kwargs[base] == X.shape[0]
return fit_orig(**kwargs)
with mock.patch.object(est.model_, "fit", new=check_batch_size):
est.fit(X, y)
def test_compile_model_from_params():
"""Tests that if build_fn returns an un-compiled model,
the __init__ parameters will be used to compile it
and that if build_fn returns a compiled model
it is not re-compiled.
"""
# Load data
data = load_boston()
X, y = data.data[:100], data.target[:100]
losses = ("mean_squared_error", "mean_absolute_error")
# build_fn that does not compile
def build_fn(compile_with_loss=None):
model = Sequential()
model.add(keras.layers.Dense(X.shape[1], input_shape=(X.shape[1], )))
model.add(keras.layers.Activation("relu"))
model.add(keras.layers.Dense(1))
model.add(keras.layers.Activation("linear"))
if compile_with_loss:
model.compile(loss=compile_with_loss)
return model
for loss in losses:
estimator = KerasRegressor(
model=build_fn,
loss=loss,
# compile_with_loss=None returns an un-compiled model
compile_with_loss=None,
)
estimator.fit(X, y)
assert estimator.model_.loss.__name__ == loss
for myloss in losses:
estimator = KerasRegressor(
model=build_fn,
loss="binary_crossentropy",
# compile_with_loss != None overrides loss
compile_with_loss=myloss,
)
estimator.fit(X, y)
assert estimator.model_.loss == myloss
def test_warm_start():
"""Test the warm start parameter."""
# Load data
data = fetch_california_housing()
X, y = data.data[:100], data.target[:100]
# Initial fit
estimator = KerasRegressor(
model=dynamic_regressor,
model__hidden_layer_sizes=(100,),
)
estimator.fit(X, y)
model = estimator.model_
# With warm start, successive calls to fit
# should NOT create a new model
estimator.set_params(warm_start=True)
estimator.fit(X, y)
assert model is estimator.model_
# Without warm start, each call to fit
# should create a new model instance
estimator.set_params(warm_start=False)
for _ in range(3):
estimator.fit(X, y)
assert model is not estimator.model_
model = estimator.model_
def test_kerasregressor_r2_as_metric_in_model():
"""Test custom R^2 implementation as part of a model"""
epochs = 25
est = KerasRegressor(
dynamic_regressor,
metrics=[KerasRegressor.r_squared],
epochs=epochs,
random_state=42,
)
y = np.random.uniform(size=(1000,))
X = y.reshape((-1, 1))
est.fit(X, y)
scores = np.array(est.history_["r_squared"])
# basic sanity check
assert np.all(scores <= 1) and len(scores) == epochs, scores
# rough estimate of expected end result given the random seed
assert scores[-1] > 0.9, scores
def test_current_epoch_property(self, warm_start, epochs_prefix):
"""Test the public current_epoch property
that tracks the overall training epochs.
The warm_start parameter should have
NO impact on this behavior.
The prefix should NOT have any impact on
behavior. It is tested because the epochs
param has special handling within param routing.
"""
data = load_boston()
X, y = data.data[:10], data.target[:10]
epochs = 2
partial_fit_iter = 3
estimator = KerasRegressor(
model=dynamic_regressor,
loss=KerasRegressor.r_squared,
model__hidden_layer_sizes=[],
warm_start=warm_start,
)
estimator.set_params(**{epochs_prefix + "epochs": epochs})
# Check that each partial_fit call trains for 1 epoch
for k in range(1, partial_fit_iter):
estimator.partial_fit(X, y)
assert estimator.current_epoch == k
# Check that fit calls still train for the number of
# epochs specified in the constructor
estimator.fit(X, y)
assert estimator.current_epoch == epochs
# partial_fit is able to resume from a non-zero epoch
estimator.partial_fit(X, y)
assert estimator.current_epoch == epochs + 1
def test_compile_model_from_params():
"""Tests that if build_fn returns an un-compiled model,
the __init__ parameters will be used to compile it
and that if build_fn returns a compiled model
it is not re-compiled.
"""
# Load data
data = fetch_california_housing()
X, y = data.data[:100], data.target[:100]
other_loss = losses_module.MeanAbsoluteError
# build_fn that does not compile
def build_fn(my_loss=None):
model = Sequential()
model.add(keras.layers.Dense(X.shape[1], input_shape=(X.shape[1],)))
model.add(keras.layers.Activation("relu"))
model.add(keras.layers.Dense(1))
model.add(keras.layers.Activation("linear"))
if my_loss is not None:
model.compile(loss=my_loss)
return model
# Calling with loss=None (or other default)
# and compiling within build_fn must work
loss_obj = other_loss()
estimator = KerasRegressor(
model=build_fn,
# compile_with_loss != None overrides loss
my_loss=loss_obj,
)
estimator.fit(X, y)
assert estimator.model_.loss is loss_obj
# Passing a value for loss AND compiling with
# the SAME loss should succeed, and the final loss
# should be the user-supplied loss
loss_obj = other_loss()
estimator = KerasRegressor(
model=build_fn,
loss=other_loss(),
# compile_with_loss != None overrides loss
my_loss=loss_obj,
)
estimator.fit(X, y)
assert estimator.model_.loss is loss_obj
# Passing a non-default value for loss AND compiling with
# a DIFFERENT loss should raise a ValueError
loss_obj = other_loss()
estimator = KerasRegressor(
model=build_fn,
loss=losses_module.CosineSimilarity(),
# compile_with_loss != None overrides loss
my_loss=loss_obj,
)
with pytest.raises(ValueError, match=" but model compiled with "):
estimator.fit(X, y)
# The ValueError should appear even if the default is != None
class DefaultLossNotNone(KerasRegressor):
def __init__(self, *args, loss=losses_module.CosineSimilarity(), **kwargs):
super().__init__(*args, **kwargs, loss=loss)
loss_obj = other_loss()
estimator = DefaultLossNotNone(
model=build_fn,
my_loss=loss_obj,
)
estimator.fit(X, y)
assert estimator.model_.loss is loss_obj
loss_obj = other_loss()
estimator = DefaultLossNotNone(
model=build_fn,
loss=losses_module.CategoricalHinge(),
my_loss=loss_obj,
)
with pytest.raises(ValueError, match=" but model compiled with "):
estimator.fit(X, y)
