def _evaluate_one(self, estimator, data_preproc, scorers):
res = []
for X_train, X_test, y_train, y_test in data_preproc:
est = clone(estimator)
est.fit(X_train, y_train)
# fit_time = time.time() - start_time
# _score will return dict if is_multimetric is True
with warnings.catch_warnings():
warnings.filterwarnings('ignore',
category=UndefinedMetricWarning)
test_scores = _multimetric_score(est, X_test, y_test, scorers)
# score_time = time.time() - start_time - fit_time
# train_scores = _multimetric_score(estimator, X_train, y_train,
# scorers)
res.append(test_scores)
res_mean = pd.DataFrame(res).mean(axis=0)
try:
# show only last step of pipeline for simplicity
name = nice_repr(estimator.steps[-1][1])
except AttributeError:
name = nice_repr(estimator)
name = name.replace("\s", " ")
if self.verbose:
print(name)
res_string = "".join("{}: {:.4f} ".format(m, s)
for m, s in res_mean.items())
print(res_string)
res_mean.name = name
self.log_.append(res_mean)
return res_mean
def _compute_metrics_train_test(self, X_tr, y_tr, X_te, y_te):
'''Compute metrics on test set, given entire train-test split'''
X_tr, y_tr = self._format_inputs(X_tr, y_tr)
X_te, y_te = self._format_inputs(X_te, y_te)
# fit model on entire training set
self.estimator.fit(X_tr, y_tr)
scorers = {
scorer_info.name: scorer_info.scorer
for scorer_info in self.scorers_info
}
multimetric_score_results = _multimetric_score(self.estimator, X_te,
y_te, scorers)
return self._process_cv_results(multimetric_score_results,
filter_testing_keys=False)
def __fit_and_score(x_train, y_train, x_test, y_test, clf, cv_repeat=None, grid_search_k=None, param_grid=None, grid_scorer=None, scorers=None):
"""Fitting and scoring a classifier through crossvalidated gridsearch over
a parameter grid.
"""
# copy classifier
clf = clone(clf)
# grid search
grid_search_clf = GridSearchCV(
estimator=clf,
param_grid=param_grid,
scoring=scorers,
cv=grid_search_k,
refit=grid_scorer,
)
grid_search_clf = grid_search_clf.fit(x_train, y_train)
# save to results
result = {}
result['gridsearch'] = grid_search_clf.cv_results_
result['best_params'] = grid_search_clf.best_params_
result['scores'] = _multimetric_score(grid_search_clf, x_test, y_test, scorers)
result['cv_repeat'] = cv_repeat
result['clf'] = clf.__class__.__name__
return result
def test__score_with_group__multimetric_score_with_group():
roc_auc_scorer = SCORERS["roc_auc"]
np.random.seed(123)
X_test = np.random.randn(100,10)
y_test = 1*(np.random.randn(100)>0)
group_test = np.array([0]*25 + [1] * 25 + [2] * 25 + [3]*25)
estimator = LogisticRegression(solver="lbfgs", random_state=123)
estimator.fit(X_test,y_test)
#######################################################
### Test with a scorer that doesn't accept group ###
#######################################################
for i in range(2):
if i == 0:
result1 = _score_with_group(estimator, X_test, y_test, None, roc_auc_scorer)
else:
result1 = _score_with_group(estimator, X_test, y_test, group_test, roc_auc_scorer)
result2 = _score(estimator, X_test, y_test, roc_auc_scorer)
assert not pd.isnull(result1)
assert isinstance(result1, numbers.Number)
assert abs(result1 - result2) <= 10**(-10)
for i in range(2):
if i == 0:
result1 = _multimetric_score_with_group(estimator, X_test, y_test, None, {"auc":roc_auc_scorer})
else:
result1 = _multimetric_score_with_group(estimator, X_test, y_test, group_test, {"auc":roc_auc_scorer})
result2 = _multimetric_score(estimator, X_test, y_test, {"auc":roc_auc_scorer})
assert isinstance(result1,dict)
assert set(result1.keys()) == {"auc"}
assert not pd.isnull(result1["auc"])
assert isinstance(result1["auc"],numbers.Number)
assert abs(result1["auc"] - result2["auc"]) <= 10**(-10)
##############################################
### test with a scorer that accepts group ###
##############################################
max_proba_scorer = _GroupProbaScorer(score_func=max_proba_group_accuracy, sign=1, kwargs={})
result1 = _score_with_group(estimator, X_test, y_test, group_test, max_proba_scorer)
assert not pd.isnull(result1)
assert isinstance(result1, numbers.Number)
assert 0 <= result1 <= 1
with pytest.raises(TypeError):
result1 = _score_with_group(estimator, X_test, y_test, None, max_proba_scorer)
# raise error because scorer expects group
result1 = _multimetric_score_with_group(estimator, X_test, y_test, group_test, {"mp_score":max_proba_scorer})
assert isinstance(result1, dict)
assert set(result1.keys()) == {"mp_score"}
r = result1["mp_score"]
assert not pd.isnull(r)
assert isinstance(r, numbers.Number)
assert 0 <= r <= 1
with pytest.raises(TypeError):
result1 = _multimetric_score_with_group(estimator, X_test, y_test, None, {"mp_score":max_proba_scorer})
# raise error because scorer expects group
#######################
### test with both ###
#######################
result1 = _multimetric_score_with_group(estimator, X_test, y_test, group_test, {"mp_score":max_proba_scorer,
"auc":roc_auc_scorer
})
assert isinstance(result1, dict)
assert set(result1.keys()) == {"auc","mp_score"}
r = result1["mp_score"]
assert not pd.isnull(r)
assert isinstance(r, numbers.Number)
assert 0 <= r <= 1