def test_with_unnecessary_target(self, teardown):
x = Input()
y_t = Input()
classifier = RandomForestClassifier()
y_p = classifier(x, y_t)
model = Model(x, y_p, y_t)
model.fit(iris.data, iris.target)
# won't require the target is trainable was set to False,
# but won't complain if it was passed to fit
classifier.trainable = False
model.fit(iris.data, iris.target)
def test_nested_model_stack(teardown):
x_data = iris.data
y_t_data = iris.target
random_state = 123
n_components = 2
# ----------- baikal way
stacked_model_baikal = make_naive_stacked_model(
n_components, random_state, x_data, y_t_data
)
y_pred_baikal = stacked_model_baikal.predict(x_data)
# ----------- traditional way
# Submodel 1
submodel1 = LogisticRegression(
multi_class="multinomial", solver="lbfgs", random_state=random_state
)
pca = PCA(n_components=n_components, random_state=random_state)
pca.fit(x_data)
pca_trans = pca.transform(x_data)
submodel1.fit(pca_trans, y_t_data)
submodel1_pred = submodel1.predict(pca_trans)
# Submodel 2 (a nested stacked model)
random_forest = RandomForestClassifier(random_state=random_state)
random_forest.fit(x_data, y_t_data)
random_forest_pred = random_forest.predict(x_data)
extra_trees = ExtraTreesClassifier(random_state=random_state)
extra_trees.fit(x_data, y_t_data)
extra_trees_pred = extra_trees.predict(x_data)
features = np.stack([random_forest_pred, extra_trees_pred], axis=1)
submodel2 = LogisticRegression(
multi_class="multinomial", solver="lbfgs", random_state=random_state
)
submodel2.fit(features, y_t_data)
submodel2_pred = submodel2.predict(features)
# Stacked model
features = np.stack([submodel1_pred, submodel2_pred], axis=1)
stacked_model_traditional = LogisticRegression(
multi_class="multinomial", solver="lbfgs", random_state=random_state
)
stacked_model_traditional.fit(features, y_t_data)
y_pred_traditional = stacked_model_traditional.predict(features)
assert_array_equal(y_pred_baikal, y_pred_traditional)
def test_fit_predict_ensemble(teardown):
mask = iris.target != 2 # Reduce to binary problem to avoid ConvergenceWarning
x_data = iris.data
y_t_data = iris.target
random_state = 123
# baikal way
x = Input()
y_t = Input()
y1 = LogisticRegression(random_state=random_state)(x, y_t)
y2 = RandomForestClassifier(random_state=random_state)(x, y_t)
features = Stack(axis=1)([y1, y2])
y = LogisticRegression(random_state=random_state)(features, y_t)
model = Model(x, y, y_t)
model.fit(x_data, y_t_data)
y_pred_baikal = model.predict(x_data)
# traditional way
logreg = sklearn.linear_model.LogisticRegression(random_state=random_state)
logreg.fit(x_data, y_t_data)
logreg_pred = logreg.predict(x_data)
random_forest = sklearn.ensemble.RandomForestClassifier(random_state=random_state)
random_forest.fit(x_data, y_t_data)
random_forest_pred = random_forest.predict(x_data)
features = np.stack([logreg_pred, random_forest_pred], axis=1)
ensemble = sklearn.linear_model.LogisticRegression(random_state=random_state)
ensemble.fit(features, y_t_data)
y_pred_traditional = ensemble.predict(features)
assert_array_equal(y_pred_baikal, y_pred_traditional)
def test_grid_search_cv_with_tunable_step():
param_grid = {
"classifier": [
LogisticRegression(random_state=random_state),
RandomForestClassifier(random_state=random_state),
],
"pca__n_components": [2, 4],
}
# baikal way
def build_fn():
x = Input()
y_t = Input()
h = PCA(random_state=random_state, name="pca")(x)
y = LogisticRegression(random_state=random_state,
name="classifier")(h, y_t)
model = Model(x, y, y_t)
return model
sk_model = SKLearnWrapper(build_fn)
gscv_baikal = GridSearchCV(
sk_model,
param_grid,
cv=cv,
scoring="accuracy",
return_train_score=True,
verbose=verbose,
)
gscv_baikal.fit(x_data, y_t_data)
# traditional way
pca = sklearn.decomposition.PCA(random_state=random_state)
classifier = sklearn.linear_model.LogisticRegression(
random_state=random_state)
pipe = Pipeline([("pca", pca), ("classifier", classifier)])
gscv_traditional = GridSearchCV(
pipe,
param_grid,
cv=cv,
scoring="accuracy",
return_train_score=True,
verbose=verbose,
)
gscv_traditional.fit(x_data, y_t_data)
assert gscv_traditional.best_params_ == gscv_baikal.best_params_
assert_array_equal(
gscv_traditional.cv_results_["mean_train_score"],
gscv_baikal.cv_results_["mean_train_score"],
)
assert_array_equal(
gscv_traditional.cv_results_["mean_test_score"],
gscv_baikal.cv_results_["mean_test_score"],
)
def test_fit_predict_naive_stack(teardown):
x_data = iris.data
y_t_data = iris.target
random_state = 123
# baikal way
x = Input()
y_t = Input()
y1 = LogisticRegression(random_state=random_state, solver="liblinear")(x, y_t)
y2 = RandomForestClassifier(random_state=random_state)(x, y_t)
features = Stack(axis=1)([y1, y2])
y = LogisticRegression(random_state=random_state, solver="liblinear")(features, y_t)
model = Model(x, y, y_t)
model.fit(x_data, y_t_data)
y_pred_baikal = model.predict(x_data)
# traditional way
logreg = LogisticRegression(random_state=random_state, solver="liblinear")
logreg.fit(x_data, y_t_data)
logreg_pred = logreg.predict(x_data)
random_forest = RandomForestClassifier(random_state=random_state)
random_forest.fit(x_data, y_t_data)
random_forest_pred = random_forest.predict(x_data)
features = np.stack([logreg_pred, random_forest_pred], axis=1)
stacked = LogisticRegression(random_state=random_state, solver="liblinear")
stacked.fit(features, y_t_data)
y_pred_traditional = stacked.predict(features)
assert_array_equal(y_pred_baikal, y_pred_traditional)
def test_get_set_params_invariance(teardown):
pca = PCA(name="pca")
classifier = RandomForestClassifier(name="classifier")
x = Input()
h = pca(x)
y = classifier(h)
model = Model(x, y)
params1 = model.get_params()
model.set_params(**params1)
params2 = model.get_params()
assert params1 == params2
def make_naive_stacked_model(n_components, random_state, x_data, y_t_data):
# An unnecessarily complex Model
# Sub-model 1
x1 = Input(name="x1")
y1_t = Input(name="y1_t")
h1 = PCA(n_components=n_components, random_state=random_state, name="pca_sub1")(x1)
y1 = LogisticRegression(
multi_class="multinomial",
solver="lbfgs",
random_state=random_state,
name="logreg_sub1",
)(h1, y1_t)
submodel1 = Model(x1, y1, y1_t, name="submodel1")
# Sub-model 2 (a nested stacked model)
x2 = Input(name="x2")
y2_t = Input(name="y2_t")
y2_1 = RandomForestClassifier(random_state=random_state, name="rforest_sub2")(
x2, y2_t
)
y2_2 = ExtraTreesClassifier(random_state=random_state, name="extrees_sub2")(
x2, y2_t
)
features = Stack(axis=1, name="stack_sub2")([y2_1, y2_2])
y2 = LogisticRegression(
multi_class="multinomial",
solver="lbfgs",
random_state=random_state,
name="logreg_sub2",
)(features, y2_t)
submodel2 = Model(x2, y2, y2_t, name="submodel2")
# Stack of submodels
x = Input(name="x")
y_t = Input(name="y_t")
y1 = submodel1(x, y_t)
y2 = submodel2(x, y_t)
features = Stack(axis=1, name="stack")([y1, y2])
y = LogisticRegression(
multi_class="multinomial",
solver="lbfgs",
random_state=random_state,
name="logreg_stacked",
)(features, y_t)
stacked_model_baikal = Model(x, y, y_t, name="stacked")
stacked_model_baikal.fit(x_data, y_t_data)
return stacked_model_baikal
def test_fit_predict_standard_stack(teardown):
# This uses the "standard" protocol where the 2nd level features
# are the out-of-fold predictions of the 1st. It also appends the
# original data to the 2nd level features.
# See for example: https://www.kdnuggets.com/2017/02/stacking-models-imropved-predictions.html
X_data, y_t_data = breast_cancer.data, breast_cancer.target
X_train, X_test, y_t_train, y_t_test = train_test_split(X_data,
y_t_data,
test_size=0.2,
random_state=0)
random_state = 42
# baikal way
x = Input()
y_t = Input()
y_p1 = RandomForestClassifierOOF(n_estimators=10,
random_state=random_state)(
x, y_t, compute_func="predict_proba")
y_p1 = Lambda(lambda array: array[:, 1:])(y_p1) # remove collinear feature
x_scaled = StandardScaler()(x)
y_p2 = LinearSVCOOF(random_state=random_state)(
x_scaled, y_t, compute_func="decision_function")
stacked_features = ColumnStack()([x, y_p1, y_p2])
y_p = LogisticRegression(solver="liblinear",
random_state=random_state)(stacked_features, y_t)
model = Model(x, y_p, y_t)
model.fit(X_train, y_t_train)
y_pred_baikal = model.predict(X_test)
# traditional way
estimators = [
("rf",
RandomForestClassifier(n_estimators=10, random_state=random_state)),
("svr",
make_pipeline(StandardScaler(),
LinearSVC(random_state=random_state))),
]
clf = sklearn.ensemble.StackingClassifier(
estimators=estimators,
final_estimator=LogisticRegression(solver="liblinear",
random_state=random_state),
passthrough=True,
)
y_pred_traditional = clf.fit(X_train, y_t_train).predict(X_test)
assert_array_equal(y_pred_baikal, y_pred_traditional)
def test_fit_predict_ensemble_with_proba_features(teardown):
mask = iris.target != 2 # Reduce to binary problem to avoid ConvergenceWarning
x_data = iris.data[mask]
y_t_data = iris.target[mask]
random_state = 123
n_estimators = 5
# baikal way
x = Input()
y_t = Input()
y1 = LogisticRegression(random_state=random_state, function="predict_proba")(x, y_t)
y2 = RandomForestClassifier(
n_estimators=n_estimators, random_state=random_state, function="apply"
)(x, y_t)
features = Concatenate(axis=1)([y1, y2])
y = LogisticRegression(random_state=random_state)(features, y_t)
model = Model(x, y, y_t)
model.fit(x_data, y_t_data)
y_pred_baikal = model.predict(x_data)
# traditional way
logreg = sklearn.linear_model.LogisticRegression(random_state=random_state)
logreg.fit(x_data, y_t_data)
logreg_proba = logreg.predict_proba(x_data)
random_forest = sklearn.ensemble.RandomForestClassifier(
n_estimators=n_estimators, random_state=random_state
)
random_forest.fit(x_data, y_t_data)
random_forest_leafidx = random_forest.apply(x_data)
features = np.concatenate([logreg_proba, random_forest_leafidx], axis=1)
ensemble = sklearn.linear_model.LogisticRegression(random_state=random_state)
ensemble.fit(features, y_t_data)
y_pred_traditional = ensemble.predict(features)
assert_array_equal(y_pred_baikal, y_pred_traditional)
def test_fit_predict_naive_stack_with_proba_features(teardown):
mask = iris.target != 2 # Reduce to binary problem to avoid ConvergenceWarning
x_data = iris.data[mask]
y_t_data = iris.target[mask]
random_state = 123
n_estimators = 5
# baikal way
x = Input()
y_t = Input()
y_p1 = LogisticRegression(random_state=random_state)(
x, y_t, compute_func="predict_proba"
)
y_p2 = RandomForestClassifier(n_estimators=n_estimators, random_state=random_state)(
x, y_t, compute_func="apply"
)
y_p1 = Lambda(compute_func=lambda array: array[:, 1:])(y_p1)
y_p2 = Lambda(compute_func=lambda array: array[:, 1:])(y_p2)
features = Concatenate(axis=1)([y_p1, y_p2])
y_p = LogisticRegression(random_state=random_state)(features, y_t)
model = Model(x, y_p, y_t)
model.fit(x_data, y_t_data)
y_pred_baikal = model.predict(x_data)
# traditional way
logreg = LogisticRegression(random_state=random_state)
logreg.fit(x_data, y_t_data)
logreg_proba = logreg.predict_proba(x_data)
random_forest = RandomForestClassifier(
n_estimators=n_estimators, random_state=random_state
)
random_forest.fit(x_data, y_t_data)
random_forest_leafidx = random_forest.apply(x_data)
features = np.concatenate(
[logreg_proba[:, 1:], random_forest_leafidx[:, 1:]], axis=1
)
stacked = LogisticRegression(random_state=random_state)
stacked.fit(features, y_t_data)
y_pred_traditional = stacked.predict(features)
assert_array_equal(y_pred_baikal, y_pred_traditional)
def test_set_params(teardown):
pca = PCA(name="pca")
classifier = RandomForestClassifier(name="classifier")
concat = Concatenate(name="concat") # a step without get_params/set_params
x = Input()
h = pca(x)
c = concat([x, h])
y = classifier(c)
model = Model(x, y)
# Fails when setting params on step that does not implement set_params
new_params_wrong = {"concat__axis": 2}
with pytest.raises(AttributeError):
model.set_params(**new_params_wrong)
# Fails when setting params on step that does not exist
new_params_wrong = {"non_existent_step__param": 42}
with pytest.raises(ValueError):
model.set_params(**new_params_wrong)
# Fails when setting a non-existent param in a step
new_params_wrong = {"pca__non_existent_param": 42}
with pytest.raises(ValueError):
model.set_params(**new_params_wrong)
new_classifier = LogisticRegression()
new_params = {
"classifier": new_classifier,
"pca__n_components": 4,
"pca__whiten": True,
"classifier__C": 100.0,
"classifier__fit_intercept": False,
"classifier__penalty": "l1",
}
model.set_params(**new_params)
params = model.get_params()
expected = {
"pca": pca,
"classifier": new_classifier,
"concat": concat,
"pca__n_components": 4,
"pca__whiten": True,
"pca__tol": 0.0,
"pca__svd_solver": "auto",
"pca__copy": True,
"pca__random_state": None,
"pca__iterated_power": "auto",
"classifier__C": 100.0,
"classifier__class_weight": None,
"classifier__dual": False,
"classifier__fit_intercept": False,
"classifier__intercept_scaling": 1,
"classifier__max_iter": 100,
"classifier__multi_class": "warn",
"classifier__n_jobs": None,
"classifier__penalty": "l1",
"classifier__random_state": None,
"classifier__solver": "warn",
"classifier__tol": 0.0001,
"classifier__verbose": 0,
"classifier__warm_start": False,
"classifier__l1_ratio": None,
}
assert expected == params