def test_incompatible_output_dimensions(): """Compares to the scikit-learn RandomForestRegressor classifier. """ # create dataset with 4 outputs X = np.random.rand(10, 20) y = np.random.randint(low=0, high=3, size=(10, 4)) # create a model with 2 outputs def build_fn_clf( meta: Dict[str, Any], compile_kwargs: Dict[str, Any], ) -> Model: """Builds a Sequential based classifier.""" model = Sequential() model.add(Dense(20, input_shape=(20, ), activation="relu")) model.add(Dense(np.unique(y).size, activation="relu")) model.compile( optimizer="sgd", loss="categorical_crossentropy", metrics=["accuracy"], ) return model clf = KerasClassifier(model=build_fn_clf) with pytest.raises(RuntimeError): clf.fit(X, y)
def test_parameter_precedence(): """Routed parameters should override non-routed parameters, and fit keyword arguments should override routed""" class TestModel(Sequential): def fit(self, *args, **kwargs): assert kwargs["class_weight"] == {0: 0.5, 1: 0.5} assert kwargs.pop("custom") == "fit_keyword" return super().fit(*args, **kwargs) def get_model() -> TestModel: return TestModel([ layers_mod.InputLayer((1, )), layers_mod.Dense(1, activation="sigmoid") ]) X, y = [[1], [2]], [0, 1] clf = KerasClassifier( get_model, loss="binary_crossentropy", fit__class_weight={ 0: 0.5, 1: 0.5, }, # test w/ a built in parameter to make sure we can override them fit__custom="constructor_routed", ) clf.fit(X, y, custom="fit_keyword")
def test_optimizer(optimizer): """Tests compiling of single optimizer with options. Since there can only ever be a single optimizer, there is no ("name", optimizer, "output") option. Only optimizer classes will be compiled with custom options, all others (class names, function names) should pass through untouched. """ # Single output X, y = make_classification() est = KerasClassifier( model=get_model, optimizer=optimizer, optimizer__learning_rate=0.15, optimizer__momentum=0.5, loss="binary_crossentropy", ) est.fit(X, y) est_opt = est.model_.optimizer if not isinstance(optimizer, str): assert float(est_opt.momentum.value()) == pytest.approx(0.5) assert float(est_opt.learning_rate) == pytest.approx(0.15, abs=1e-6) else: est_opt.__class__ == optimizers_module.get(optimizer).__class__
def test_compiling_of_routed_parameters(): """Tests that routed parameters can themselves be compiled. """ X, y = make_classification() class Foo: got = dict() def __init__(self, foo_kwarg="foo_kwarg_default"): self.foo_kwarg = foo_kwarg class MyLoss(losses_module.Loss): def __init__(self, param1="param1_default", *args, **kwargs): super().__init__(*args, **kwargs) self.param1 = param1 def __call__(self, y_true, y_pred, sample_weight=None): return losses_module.binary_crossentropy(y_true, y_pred) est = KerasClassifier( model=get_model, loss=MyLoss, loss__param1=[Foo, Foo], loss__param1__foo_kwarg=1, loss__param1__0__foo_kwarg=2, ) est.fit(X, y) assert est.model_.loss.param1[0].foo_kwarg == 2 assert est.model_.loss.param1[1].foo_kwarg == 1
def test_compiling_of_routed_parameters(): """Tests that routed parameters can themselves be compiled. """ X, y = make_classification() class Foo: got = dict() def __init__(self, foo_kwarg): self.foo_kwarg = foo_kwarg class MyLoss: def __init__(self, param1): self.param1 = param1 self.__name__ = str(id(self)) def __call__(self, y_true, y_pred): return losses_module.binary_crossentropy(y_true, y_pred) est = KerasClassifier( model=get_model, loss=MyLoss, loss__param1=[Foo, Foo], loss__param1__foo_kwarg=1, loss__param1__0__foo_kwarg=2, ) est.fit(X, y) assert est.model_.loss.param1[0].foo_kwarg == 2 assert est.model_.loss.param1[1].foo_kwarg == 1
def test_invalid_build_fn(self): class Model: pass clf = KerasClassifier(model=Model()) with pytest.raises(TypeError, match="``model`` must be"): clf.fit(np.array([[0], [1]]), np.array([0, 1]))
def test_build_fn_deprecation(): """An appropriate warning is raised when using the `build_fn` parameter instead of `model`. """ clf = KerasClassifier(build_fn=dynamic_classifier, model__hidden_layer_sizes=(100,)) with pytest.warns(UserWarning, match="``build_fn`` will be renamed to ``model``"): clf.fit([[0], [1]], [0, 1])
def test_incompatible_output_dimensions(): """Compares to the scikit-learn RandomForestRegressor classifier. """ # create dataset with 4 outputs X = np.random.rand(10, 20) y = np.random.randint(low=0, high=3, size=(10,)) # create a model with 2 outputs def build_fn_clf(meta: Dict[str, Any], compile_kwargs: Dict[str, Any],) -> Model: # get params n_features_in_ = meta["n_features_in_"] inp = Input((n_features_in_,)) x1 = Dense(100)(inp) binary_out = Dense(1, activation="sigmoid")(x1) cat_out = Dense(2, activation="softmax")(x1) model = Model([inp], [binary_out, cat_out]) model.compile(loss=["binary_crossentropy", "categorical_crossentropy"]) return model clf = KerasClassifier(model=build_fn_clf) with pytest.raises(ValueError, match="input of size"): clf.fit(X, y)
def test_metrics(self, metric): """Test the metrics param. Specifically test ``accuracy``, which Keras automatically matches to the loss function and hence should be passed through as a string and not as a retrieved function. """ est = KerasClassifier(model=dynamic_classifier, model__hidden_layer_sizes=(100, ), metrics=[metric]) X, y = make_classification() est.fit(X, y) assert len(est.history_[metric]) == 1
def test_class_weight_balanced(class_weight): """KerasClassifier should accept the class_weight parameter in the same format as ScikitLearn. Passing "balanced" will automatically compute class_weight. Class weights will always be converted to sample weights before calling the Keras model, preserving compatibility with encoders. """ clf = KerasClassifier(model=dynamic_classifier, model__hidden_layer_sizes=[], class_weight="balanced") clf.fit([[1], [1]], [0, 1]) class TestModel(Sequential): def fit(self, *args, **kwargs): np.testing.assert_equal( kwargs["sample_weight"] / kwargs["sample_weight"], [1, 1]) return super().fit(*args, **kwargs) def get_model() -> TestModel: return TestModel([ layers_mod.InputLayer((1, )), layers_mod.Dense(1, activation="sigmoid") ]) X, y = [[1], [1]], [0, 1] clf = KerasClassifier(get_model, loss="binary_crossentropy", class_weight=class_weight) clf.fit(X, y)
def test_loss_routed_params_iterable(loss, n_outputs_): """Tests compiling of loss when it is given as an iterable of losses mapping to outputs. """ X, y = make_classification() y = np.column_stack([y for _ in range(n_outputs_)]).squeeze() # Test iterable with global routed param est = KerasClassifier( model=get_model, loss=[loss], loss__from_logits=True, # default is False ) est.fit(X, y) assert est.model_.loss[0].from_logits # Test iterable with index-based routed param est = KerasClassifier( model=get_model, loss=[loss], loss__from_logits=True, loss__0__from_logits=False, # should override above ) est.fit(X, y) assert est.model_.loss[0].from_logits == False
def test_loss_routed_params_dict(loss, n_outputs_): """Tests compiling of loss when it is given as an dict of losses mapping to outputs. """ X, y = make_classification() y = np.column_stack([y for _ in range(n_outputs_)]).squeeze() # Test dict with global routed param est = KerasClassifier( model=get_model, loss={"out1": loss}, loss__from_logits=True, # default is False ) est.fit(X, y) assert est.model_.loss["out1"].from_logits == True # Test dict with key-based routed param est = KerasClassifier( model=get_model, loss={"out1": loss}, loss__from_logits=True, loss__out1__from_logits=False, # should override above ) est.fit(X, y) assert est.model_.loss["out1"].from_logits == False
def test_callback_compiling_args_or_kwargs(): """Test compiling callbacks with routed positional (args) or keyword (kwargs) arguments.""" def get_clf() -> keras.Model: model = keras.models.Sequential() model.add(keras.layers.InputLayer((1, ))) model.add(keras.layers.Dense(1, activation="sigmoid")) return model class ArgsOnlyCallback(keras.callbacks.Callback): def __init__(self, *args): assert args == ("arg0", "arg1") ArgsOnlyCallback.called = True super().__init__() class KwargsOnlyCallback(keras.callbacks.Callback): def __init__(self, **kwargs): assert kwargs == {"kwargname": None} KwargsOnlyCallback.called = True super().__init__() class ArgsAndKwargsCallback(keras.callbacks.Callback): def __init__(self, *args, **kwargs): assert args == ("arg", ) assert kwargs == {"kwargname": None} ArgsAndKwargsCallback.called = True super().__init__() clf = KerasClassifier( model=get_clf, epochs=5, optimizer=keras.optimizers.SGD, optimizer__learning_rate=0.1, loss="binary_crossentropy", callbacks={ "args": ArgsOnlyCallback, "kwargs": KwargsOnlyCallback, "argskwargs": ArgsAndKwargsCallback, }, callbacks__args__1="arg1", # passed as an arg callbacks__args__0= "arg0", # unorder the args on purpose, SciKeras should not care about the order of the keys callbacks__kwargs__kwargname=None, # passed as a kwarg callbacks__argskwargs__0="arg", # passed as an arg callbacks__argskwargs__kwargname=None, # passed as a kwarg ) clf.fit([[1]], [1]) for cls in (ArgsOnlyCallback, KwargsOnlyCallback, ArgsAndKwargsCallback): assert cls.called
def test_loss_invalid_string(): """Tests that a ValueError is raised when an unknown string is passed as a loss. """ X, y = make_classification() loss = "binary_crossentropr" # binary_crossentropr is not a loss est = KerasClassifier( model=get_model, num_hidden=20, loss=loss, ) with pytest.raises(ValueError, match="Unknown loss function"): est.fit(X, y)
def test_loss(loss, n_outputs_): """Tests compiling of single loss using routed parameters. """ X, y = make_classification() y = np.column_stack([y for _ in range(n_outputs_)]).squeeze() est = KerasClassifier( model=get_model, loss=loss, loss__name="custom_name", ) est.fit(X, y) assert str(loss) in str(est.model_.loss) or isinstance( est.model_.loss, loss)
def test_optimizer_invalid_string(): """Tests that a ValueError is raised when an unknown string is passed as an optimizer. """ X, y = make_classification() optimizer = "sgf" # sgf is not a loss est = KerasClassifier( model=get_model, optimizer=optimizer, loss="binary_crossentropy", ) with pytest.raises(ValueError, match="Unknown optimizer"): est.fit(X, y)
def test_keras(c, s, a, b): # Mirror the mnist dataset X, y = make_classification(n_classes=10, n_features=784, n_informative=100) X = X.astype("float32") assert y.dtype == np.dtype("int64") model = KerasClassifier(build_fn=_keras_build_fn, lr=0.01, verbose=False) params = {"lr": loguniform(1e-3, 1e-1)} search = IncrementalSearchCV(model, params, max_iter=3, n_initial_parameters=5, decay_rate=None) yield search.fit(X, y) # search.fit(X, y) assert search.best_score_ >= 0 # Make sure the model trains, and scores aren't constant scores = { ident: [h["score"] for h in hist] for ident, hist in search.model_history_.items() } assert all(len(hist) == 3 for hist in scores.values()) nuniq_scores = [pd.Series(v).nunique() for v in scores.values()] assert max(nuniq_scores) > 1
def test_metrics_two_metric_per_output(n_outputs_): """Metrics without the ("name", metric, "output") syntax should ignore all routed and custom options. This tests multiple (two) metrics per output. """ X, y = make_classification() y = np.column_stack([y for _ in range(n_outputs_)]).squeeze() metric_class = metrics_module.BinaryAccuracy # loss functions for each output and joined show up as metrics metric_idx = 1 + (n_outputs_ if n_outputs_ > 1 else 0) # List of lists of metrics if n_outputs_ == 1: metrics_ = [metric_class(name="1"), metric_class(name="2")] else: metrics_ = [[metric_class(name="1"), metric_class(name="2")] for _ in range(n_outputs_)] est = KerasClassifier( model=get_model, loss="binary_crossentropy", metrics=metrics_, ) est.fit(X, y) if n_outputs_ == 1: assert est.model_.metrics[metric_idx].name == "1" else: # For multi-output models, Keras pre-appends the output name assert est.model_.metrics[metric_idx].name == "out1_1" # List of lists of metrics if n_outputs_ == 1: metrics_ = {"out1": [metric_class(name="1"), metric_class(name="2")]} else: metrics_ = { f"out{i+1}": [metric_class(name="1"), metric_class(name="2")] for i in range(n_outputs_) } # Dict of metrics est = KerasClassifier( model=get_model, loss="binary_crossentropy", metrics=metrics_, ) est.fit(X, y) if n_outputs_ == 1: assert est.model_.metrics[metric_idx].name == "1" else: # For multi-output models, Keras pre-appends the output name assert est.model_.metrics[metric_idx].name == "out1_1"
def test_sample_weights_all_zero(): """Checks for a user-friendly error when sample_weights are all zero. """ # build estimator estimator = KerasClassifier( model=dynamic_classifier, model__hidden_layer_sizes=(100,), ) # we create 20 points n, d = 50, 4 X = np.random.uniform(size=(n, d)) y = np.random.choice(2, size=n).astype("uint8") sample_weight = np.zeros(y.shape) with pytest.raises(ValueError, match="only zeros were passed in sample_weight"): estimator.fit(X, y, sample_weight=sample_weight)
def test_calibratedclassifiercv(self, config): """Tests compatibility with Scikit-learn's calibrated classifier CV.""" loader, _, build_fn, _ = CONFIG[config] base_estimator = KerasClassifier( build_fn, epochs=1, model__hidden_layer_sizes=[] ) estimator = CalibratedClassifierCV(base_estimator=base_estimator, cv=5) basic_checks(estimator, loader)
def test_class_weight_param(): """Backport of sklearn.utils.estimator_checks.check_class_weight_classifiers for sklearn <= 0.23.0. """ clf = KerasClassifier( model=dynamic_classifier, model__hidden_layer_sizes=(100, ), epochs=50, random_state=0, ) problems = (2, 3) for n_centers in problems: # create a very noisy dataset X, y = make_blobs(centers=n_centers, random_state=0, cluster_std=20) X_train, X_test, y_train, _ = train_test_split(X, y, test_size=0.5, random_state=0) n_centers = len(np.unique(y_train)) if n_centers == 2: class_weight = {0: 1000, 1: 0.0001} else: class_weight = {0: 1000, 1: 0.0001, 2: 0.0001} clf.set_params(class_weight=class_weight) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) assert np.mean(y_pred == 0) > 0.87
def test_callback_param_routing_syntax(callback_kwargs: Dict[str, Any]): """Test support for the various parameter routing syntaxes for callbacks.""" def get_clf() -> keras.Model: model = keras.models.Sequential() model.add(keras.layers.InputLayer((1, ))) model.add(keras.layers.Dense(1, activation="sigmoid")) return model clf = KerasClassifier( model=get_clf, epochs=5, loss="binary_crossentropy", metrics="acc", **callback_kwargs, ) clf.fit([[1], [1]], [0, 1]) # should early stop after 1-2 epochs (depending on the TF version) since we set the accuracy delta to 1 assert clf.current_epoch < 5
def test_metrics_invalid_string(): """Tests that a ValueError is raised when an unknown string is passed as a metric. """ X, y = make_classification() metrics = [ "acccuracy", ] # acccuracy (extra `c`) is not a metric est = KerasClassifier( model=get_model, loss="binary_crossentropy", metrics=metrics, ) with pytest.raises(ValueError, match="Unknown metric function"): est.fit(X, y)
def test_sample_weights_all_zero(): """Checks for a user-friendly error when sample_weights are all zero. """ # build estimator estimator = KerasClassifier( model=dynamic_classifier, model__hidden_layer_sizes=(100, ), ) # we create 20 points n, d = 50, 4 X = np.random.uniform(size=(n, d)) y = np.random.uniform(size=n) sample_weight = np.zeros(y.shape) with pytest.raises(RuntimeError, match="no samples left"): estimator.fit(X, y, sample_weight=sample_weight)
def test_loss_uncompilable(): """Tests that a TypeError is raised when a loss that is not compilable is passed routed parameters. """ X, y = make_classification() loss = losses_module.binary_crossentropy est = KerasClassifier( model=get_model, loss=loss, loss__from_logits=True, ) with pytest.raises( TypeError, match="does not accept parameters because it's not a class"): est.fit(X, y)
def test_X_dtype_changes_incremental_fit(): X = np.array([[1, 2], [2, 3]]) y = np.array([1, 3]) est = KerasClassifier(model=dynamic_classifier, hidden_layer_sizes=(100,)) est.fit(X, y) est.partial_fit(X.astype(np.uint8), y) with pytest.raises( ValueError, match="Got `X` with dtype", ): est.partial_fit(X.astype(np.float64), y)
def test_exclude_parameters_with_further_routing(): """SciKeras should only route parameters to final destinations that do not contain further routing For example, optimizer__xyz__abc should _not_ be passed to the Optimizer as Optimizer(xyz__abc=xyz__abc). """ def get_model() -> Sequential: return Sequential([ layers_mod.InputLayer((1, )), layers_mod.Dense(1, activation="sigmoid") ]) X, y = [[1], [2]], [0, 1] clf = KerasClassifier( get_model, loss="binary_crossentropy", optimizer__this_should_not_pass__abc="error!", ) clf.fit(X, y)
def test_target_classes_change_incremental_fit(): X = np.array([[1, 2], [2, 3]]) y = np.array([1, 3]) est = KerasClassifier(model=dynamic_classifier, hidden_layer_sizes=(100,)) est.fit(X, y) est.partial_fit(X.astype(np.uint8), y) with pytest.raises( ValueError, match="Found unknown categories", ): y[0] = 10 est.partial_fit(X, y)
def test_metrics_uncompilable(): """Tests that a TypeError is raised when a metric that is not compilable is passed routed parameters. """ X, y = make_classification() metrics = [ metrics_module.get("accuracy"), ] # a function est = KerasClassifier( model=get_model, loss="binary_crossentropy", metrics=metrics, metrics__name="custom_name", ) with pytest.raises( TypeError, match="does not accept parameters because it's not a class"): est.fit(X, y)
def test_single_output_multilabel_indicator(): """Tests a target that a multilabel-indicator target can be used without errors. """ X = np.random.random(size=(100, 2)) y = np.random.randint(0, 1, size=(100, 3)) y[0, :] = 1 # i.e. not "one hot encoded" def build_fn(): model = Sequential() model.add(Dense(10, input_shape=(2, ), activation="relu")) model.add(Dense(3, activation="sigmoid")) return model clf = KerasClassifier( model=build_fn, loss="categorical_crossentropy", ) # check that there are no errors clf.fit(X, y) clf.predict(X) # check the target type assert clf.target_type_ == "multilabel-indicator" # check classes np.testing.assert_equal(clf.classes_, np.arange(3))