def cross_validate_model(self, X, y, model, num_folds, score):
'''
Realiza la validación de un modelo, con los dataset X e y, haciendo CV de numfolds
y devolviendo la metrica score.
Parametros
----------
X: numpy.array
Datos de las instancias del dataset con el que entrenar.
Y: numpy.array
Targets de las instancias de X.
model: scikit_model
modelo al que entrenar
num_folds: int
numero de folds de la validacion cruzada
score: string
métrica que devuelve, de los strings disponibles en valido en sklearn.metrics.SCORERS.keys().
Return
-------
array de resultados
array con los resultados de cada fold de la CV
'''
if score not in SCORERS.keys():
raise AttributeError(
"Atributo score debe ser válido. Ver válidos en sklearn.metrics.SCORERS.keys()"
)
print('\t' + str(model)[:20], end=' - ')
mod_scores = cross_val_score(model, X, y, cv=num_folds, scoring=score)
print('FM')
return np.array(mod_scores)
def optimize(self):
# TODO: don't hard code this!
self.model_ = Pipeline(steps=[(
"featurizer",
ColumnTransformer([
("count_vectorizer", CountVectorizer(), [0]),
])), ("estimator", SGDClassifier(warm_start=True, loss="log"))])
# TODO: also don't hard code this!
scorer = SCORERS.get("roc_auc")
print(
f"optimizing model {self.model_} with evaluation metric {scorer}")
with open(self.data) as f:
dataset = [self._parser_fn(line) for line in f.readlines()]
X, y = self.batch_to_vectors(dataset)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=self._test_size)
self.model_.fit(X_train, y_train)
train_score = scorer(self.model_, X_train, y_train)
test_score = scorer(self.model_, X_test, y_test)
print(f"train score: {train_score}")
print(f"test score: {test_score}")
joblib.dump(self.model_, self.model_path)
return self.model_
def _fit_by_cross_validation(self, X, y, number_of_splits: int = 5, label_name: str = None, cores_for_training: int = 1,
optimization_metric: str = "balanced_accuracy"):
model = self._get_ml_model()
scoring = optimization_metric
if optimization_metric not in SCORERS.keys():
scoring = "balanced_accuracy"
warnings.warn(
f"{self.__class__.__name__}: specified optimization metric ({optimization_metric}) is not defined as a sklearn scoring function, using {scoring} instead... ")
if not self.show_warnings:
warnings.simplefilter("ignore")
os.environ["PYTHONWARNINGS"] = "ignore"
self.model = RandomizedSearchCV(model, param_distributions=self._parameter_grid, cv=number_of_splits, n_jobs=cores_for_training,
scoring=scoring, refit=True)
self.model.fit(X, y)
if not self.show_warnings:
del os.environ["PYTHONWARNINGS"]
warnings.simplefilter("always")
self.model = self.model.best_estimator_ # do not leave RandomSearchCV object to be in models, use the best estimator instead
return self.model
def test_scorer_sample_weight():
"""Test that scorers support sample_weight or raise sensible errors"""
# Unlike the metrics invariance test, in the scorer case it's harder
# to ensure that, on the classifier output, weighted and unweighted
# scores really should be unequal.
X, y = make_classification(random_state=0)
_, y_ml = make_multilabel_classification(n_samples=X.shape[0],
return_indicator=True,
random_state=0)
split = train_test_split(X, y, y_ml, random_state=0)
X_train, X_test, y_train, y_test, y_ml_train, y_ml_test = split
sample_weight = np.ones_like(y_test)
sample_weight[:10] = 0
# get sensible estimators for each metric
sensible_regr = DummyRegressor(strategy='median')
sensible_regr.fit(X_train, y_train)
sensible_clf = DecisionTreeClassifier(random_state=0)
sensible_clf.fit(X_train, y_train)
sensible_ml_clf = DecisionTreeClassifier(random_state=0)
sensible_ml_clf.fit(X_train, y_ml_train)
estimator = dict([(name, sensible_regr) for name in REGRESSION_SCORERS] +
[(name, sensible_clf) for name in CLF_SCORERS] +
[(name, sensible_ml_clf)
for name in MULTILABEL_ONLY_SCORERS])
for name, scorer in SCORERS.items():
if name in MULTILABEL_ONLY_SCORERS:
target = y_ml_test
else:
target = y_test
try:
weighted = scorer(estimator[name],
X_test,
target,
sample_weight=sample_weight)
ignored = scorer(estimator[name], X_test[10:], target[10:])
unweighted = scorer(estimator[name], X_test, target)
assert_not_equal(weighted,
unweighted,
msg="scorer {0} behaves identically when "
"called with sample weights: {1} vs "
"{2}".format(name, weighted, unweighted))
assert_almost_equal(weighted,
ignored,
err_msg="scorer {0} behaves differently when "
"ignoring samples and setting sample_weight to"
" 0: {1} vs {2}".format(
name, weighted, ignored))
except TypeError as e:
assert_true(
"sample_weight" in str(e),
"scorer {0} raises unhelpful exception when called "
"with sample weights: {1}".format(name, str(e)))
def score_options():
'''
Return a list of possible scorers for a regression model
'''
from sklearn.metrics import SCORERS
score_types = sorted(SCORERS.keys())
print('Possible scores to choose from: ')
for s in score_types:
print(s)
def test_scorer_sample_weight():
# Test that scorers support sample_weight or raise sensible errors
# Unlike the metrics invariance test, in the scorer case it's harder
# to ensure that, on the classifier output, weighted and unweighted
# scores really should be unequal.
X, y = make_classification(random_state=0)
_, y_ml = make_multilabel_classification(n_samples=X.shape[0], random_state=0)
split = train_test_split(X, y, y_ml, random_state=0)
X_train, X_test, y_train, y_test, y_ml_train, y_ml_test = split
sample_weight = np.ones_like(y_test)
sample_weight[:10] = 0
# get sensible estimators for each metric
sensible_regr = DummyRegressor(strategy="median")
sensible_regr.fit(X_train, y_train)
sensible_clf = DecisionTreeClassifier(random_state=0)
sensible_clf.fit(X_train, y_train)
sensible_ml_clf = DecisionTreeClassifier(random_state=0)
sensible_ml_clf.fit(X_train, y_ml_train)
estimator = dict(
[(name, sensible_regr) for name in REGRESSION_SCORERS]
+ [(name, sensible_clf) for name in CLF_SCORERS]
+ [(name, sensible_ml_clf) for name in MULTILABEL_ONLY_SCORERS]
)
for name, scorer in SCORERS.items():
if name in MULTILABEL_ONLY_SCORERS:
target = y_ml_test
else:
target = y_test
try:
weighted = scorer(estimator[name], X_test, target, sample_weight=sample_weight)
ignored = scorer(estimator[name], X_test[10:], target[10:])
unweighted = scorer(estimator[name], X_test, target)
assert_not_equal(
weighted,
unweighted,
msg="scorer {0} behaves identically when "
"called with sample weights: {1} vs "
"{2}".format(name, weighted, unweighted),
)
assert_almost_equal(
weighted,
ignored,
err_msg="scorer {0} behaves differently when "
"ignoring samples and setting sample_weight to"
" 0: {1} vs {2}".format(name, weighted, ignored),
)
except TypeError as e:
assert_true(
"sample_weight" in str(e),
"scorer {0} raises unhelpful exception when called " "with sample weights: {1}".format(name, str(e)),
)
def evaluation_p(population: pd.DataFrame,
list_caracteres: np.array,
data: pd.DataFrame,
target: pd.Series,
model: Any,
scorer: str,
n_cv: int = 5,
sort_scores: bool = True) -> pd.DataFrame:
"""
Évaluation des individus d'une population
:param population: ensemble des individus à évaluer
:param list_caracteres: liste des caractères qui peuvent être exprimés
:param data: données à utiliser pour l'évaluation
:param target: sortie à prédire
:param model: modèle à ajuster
:param scorer: score de performance à maximiser
:param n_cv: nombre de plis pour validation croisée, doit être au moins de 2
:param sort_scores: appliquer un tri (descendant) ou non
:return:
population avec score de performance obtenu par validation croisée
"""
# TODO: voir pour passer directement la fonction scorer=make_score()
# et les paramètres associés pour ne pas la définir en dur
population_eval = population.copy()
mean_scores = []
xval_strategy = KFold(n_cv, shuffle=True, random_state=123)
for indiv in population_eval.values:
lstcols = list_caracteres[indiv]
if scorer in SCORERS.keys():
scoring_function = scorer
elif scorer == 'bic':
kwargs = {'k': len(lstcols)}
scoring_function = make_scorer(get_bic,
greater_is_better=False,
**kwargs)
else:
print(f'Scorer {scorer} unknown')
break
scores = cross_val_score(model,
data[lstcols],
target,
cv=xval_strategy,
scoring=scoring_function,
n_jobs=-1)
mean_scores.append(scores.mean())
population_scores = pd.Series(mean_scores,
index=population_eval.index,
name='score')
population_eval['score'] = population_scores
if sort_scores:
population_eval = population_eval.sort_values(by='score',
ascending=False)
else:
population_eval = population_eval.reset_index(drop=True)
return population_eval
def test_classification_scorer_sample_weight():
# Test that classification scorers support sample_weight or raise sensible
# errors
# Unlike the metrics invariance test, in the scorer case it's harder
# to ensure that, on the classifier output, weighted and unweighted
# scores really should be unequal.
X, y = make_classification(random_state=0)
_, y_ml = make_multilabel_classification(n_samples=X.shape[0], random_state=0)
split = train_test_split(X, y, y_ml, random_state=0)
X_train, X_test, y_train, y_test, y_ml_train, y_ml_test = split
sample_weight = np.ones_like(y_test)
sample_weight[:10] = 0
# get sensible estimators for each metric
estimator = _make_estimators(X_train, y_train, y_ml_train)
for name, scorer in SCORERS.items():
if name in REGRESSION_SCORERS:
# skip the regression scores
continue
if name == "top_k_accuracy":
# in the binary case k > 1 will always lead to a perfect score
scorer._kwargs = {"k": 1}
if name in MULTILABEL_ONLY_SCORERS:
target = y_ml_test
else:
target = y_test
try:
weighted = scorer(
estimator[name], X_test, target, sample_weight=sample_weight
)
ignored = scorer(estimator[name], X_test[10:], target[10:])
unweighted = scorer(estimator[name], X_test, target)
assert weighted != unweighted, (
f"scorer {name} behaves identically when called with "
f"sample weights: {weighted} vs {unweighted}"
)
assert_almost_equal(
weighted,
ignored,
err_msg=(
f"scorer {name} behaves differently "
"when ignoring samples and setting "
f"sample_weight to 0: {weighted} vs {ignored}"
),
)
except TypeError as e:
assert "sample_weight" in str(e), (
f"scorer {name} raises unhelpful exception when called "
f"with sample weights: {str(e)}"
)
def cv(self):
data = input("Predictors Name: ")
target = input("Target: ")
scaler = input(
"Input type of Problem: Regression = R, Classification = C")
no_cv = input("k-folds, k = ")
from sklearn.metrics import SCORERS
list(SCORERS.keys())
reg_scorers = [
'r2', 'neg_median_absolute_error', 'neg_mean_absolute_error',
'neg_mean_squared_error', 'neg_mean_squared_log_error',
'explained_variance'
]
class_scorers = ['precision', 'recall', 'f1', 'accuracy', 'roc_auc']
step2 = "Get scorers to cross validate on. Please separate scorers by a comma only. "
print(step2)
if scaler == "R":
metrics = input(", ".join(reg_scorers))
elif scaler == "C":
metrics = input(", ".join(class_scorers))
scorers = [i.strip() for i in metrics.split(",")]
print(self.get_script)
print("\n_____________Copy from Here_____________\n")
step1 = "# Import module to cross-validate"
print(step1)
module_import = "from sklearn.model_selection import cross_validate"
print(module_import + "\n")
step3 = f"#Create base model to cross validate "
print(step3)
print(f"model = ...\n")
step4 = f"#Define the scorers to validate on"
print(step4)
print(f"scorers = {scorers}")
step4 = f"#Cross validate model on {data}"
print(step4)
print(
f"scores = cross_validate(model, X = {data}, y = {target}, scoring = scorers)\n",
cv=no_cv)
step5 = f"#Check performance on {data}"
print(step5)
str = r"print(f'Performance : { scores }')"
print(str)
def test_scorer_sample_weight():
# Test that scorers support sample_weight or raise sensible errors
# Unlike the metrics invariance test, in the scorer case it's harder
# to ensure that, on the classifier output, weighted and unweighted
# scores really should be unequal.
X, y = make_classification(random_state=0)
_, y_ml = make_multilabel_classification(n_samples=X.shape[0],
random_state=0)
split = train_test_split(X, y, y_ml, random_state=0)
X_train, X_test, y_train, y_test, y_ml_train, y_ml_test = split
sample_weight = np.ones_like(y_test)
sample_weight[:10] = 0
# get sensible estimators for each metric
estimator = _make_estimators(X_train, y_train, y_ml_train)
for name, scorer in SCORERS.items():
if name in MULTILABEL_ONLY_SCORERS:
target = y_ml_test
else:
target = y_test
if name in REQUIRE_POSITIVE_Y_SCORERS:
target = _require_positive_y(target)
try:
weighted = scorer(estimator[name],
X_test,
target,
sample_weight=sample_weight)
ignored = scorer(estimator[name], X_test[10:], target[10:])
unweighted = scorer(estimator[name], X_test, target)
assert weighted != unweighted, (
"scorer {0} behaves identically when "
"called with sample weights: {1} vs "
"{2}".format(name, weighted, unweighted))
assert_almost_equal(weighted,
ignored,
err_msg="scorer {0} behaves differently when "
"ignoring samples and setting sample_weight to"
" 0: {1} vs {2}".format(
name, weighted, ignored))
except TypeError as e:
assert "sample_weight" in str(e), (
"scorer {0} raises unhelpful exception when called "
"with sample weights: {1}".format(name, str(e)))
def parse_cfg(self):
transformations = self.cfg['dataset'].get('transform')
if transformations:
self.cfg['dataset']['transform'] = [
eval(t) for t in transformations
]
target_transform = self.cfg['dataset'].get('target_transform')
if target_transform:
self.cfg['dataset']['target_transform'] = eval(target_transform)
scorers = self.cfg['training'].get('scorers')
if isinstance(scorers, str):
scorers = [scorers]
scorers_dict = {}
for s in scorers:
scorers_dict[s] = s if s in SCORERS.keys() else make_scorer(
eval(s))
self.cfg['training']['scorers'] = scorers_dict
models = self.cfg['training'].get('models')
if isinstance(models, list):
self.cfg['training']['models'] = [
Pipeline([(e, _get_model(e)) for e in model])
for model in models
]
elif isinstance(models, str):
self.cfg['training']['models'] = jb_load(models)
self.cfg['training']['cfg_path'] = self.cfg_path
if 'holdout' in self.cfg:
self.cfg['holdout']['cfg_path'] = self.cfg_path
scorers = self.cfg['holdout'].get('scorers')
if scorers:
valid_scorers = {
'balanced_accuracy': balanced_accuracy_score,
'accuracy': accuracy_score,
'roc_auc': roc_auc_score,
'recall': recall_score,
'specificity': specificity,
}
self.cfg['holdout']['scorers'] = [
valid_scorers[s] for s in scorers
]
def test_regression_scorer_sample_weight():
# Test that regression scorers support sample_weight or raise sensible
# errors
# Odd number of test samples req for neg_median_absolute_error
X, y = make_regression(n_samples=101, n_features=20, random_state=0)
y = _require_positive_y(y)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
sample_weight = np.ones_like(y_test)
# Odd number req for neg_median_absolute_error
sample_weight[:11] = 0
reg = DecisionTreeRegressor(random_state=0)
reg.fit(X_train, y_train)
for name, scorer in SCORERS.items():
if name not in REGRESSION_SCORERS:
# skip classification scorers
continue
try:
weighted = scorer(reg, X_test, y_test, sample_weight=sample_weight)
ignored = scorer(reg, X_test[11:], y_test[11:])
unweighted = scorer(reg, X_test, y_test)
assert weighted != unweighted, (
f"scorer {name} behaves identically when called with "
f"sample weights: {weighted} vs {unweighted}"
)
assert_almost_equal(
weighted,
ignored,
err_msg=(
f"scorer {name} behaves differently "
"when ignoring samples and setting "
f"sample_weight to 0: {weighted} vs {ignored}"
),
)
except TypeError as e:
assert "sample_weight" in str(e), (
f"scorer {name} raises unhelpful exception when called "
f"with sample weights: {str(e)}"
)
def test_scorer_sample_weight():
"""Test that scorers support sample_weight or raise sensible errors"""
# Unlike the metrics invariance test, in the scorer case it's harder
# to ensure that, on the classifier output, weighted and unweighted
# scores really should be unequal.
X, y = make_classification(random_state=0)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
sample_weight = np.ones_like(y_test)
sample_weight[:10] = 0
# get sensible estimators for each metric
sensible_regr = DummyRegressor(strategy='median')
sensible_regr.fit(X_train, y_train)
sensible_clf = DecisionTreeClassifier()
sensible_clf.fit(X_train, y_train)
estimator = dict([(name, sensible_regr)
for name in REGRESSION_SCORERS] +
[(name, sensible_clf)
for name in CLF_SCORERS])
for name, scorer in SCORERS.items():
try:
weighted = scorer(estimator[name], X_test, y_test,
sample_weight=sample_weight)
ignored = scorer(estimator[name], X_test[10:], y_test[10:])
unweighted = scorer(estimator[name], X_test, y_test)
assert_not_equal(weighted, unweighted,
"scorer {0} behaves identically when called with "
"sample weights: {1} vs {2}".format(name,
weighted,
unweighted))
assert_equal(weighted, ignored,
"scorer {0} behaves differently when ignoring "
"samples and setting sample_weight to 0: "
"{1} vs {2}".format(name, weighted, ignored))
except TypeError as e:
assert_true("sample_weight" in str(e),
"scorer {0} raises unhelpful exception when called "
"with sample weights: {1}".format(name, str(e)))
def test_scorer_memmap_input():
# Non-regression test for #6147: some score functions would
# return singleton memmap when computed on memmap data instead of scalar
# float values.
for name in SCORERS.keys():
yield check_scorer_memmap, name
from .version import __version__, VERSION
__all__ = ['Learner', 'load_examples', 'kappa', 'kendall_tau', 'spearman',
'pearson', 'f1_score_least_frequent', 'run_configuration',
'run_ablation', 'write_feature_file', 'convert_examples']
# Add our scorers to the sklearn dictionary here so that they will always be
# available if you import anything from skll
_scorers = {'f1_score_micro': make_scorer(f1_score, average='micro',
pos_label=None),
'f1_score_macro': make_scorer(f1_score, average='macro',
pos_label=None),
'f1_score_weighted': make_scorer(f1_score, average='weighted',
pos_label=None),
'f1_score_least_frequent': make_scorer(f1_score_least_frequent),
'pearson': make_scorer(pearson),
'spearman': make_scorer(spearman),
'kendall_tau': make_scorer(kendall_tau),
'unweighted_kappa': make_scorer(kappa),
'quadratic_weighted_kappa': make_scorer(kappa,
weights='quadratic'),
'linear_weighted_kappa': make_scorer(kappa, weights='linear'),
'qwk_off_by_one': make_scorer(kappa, weights='quadratic',
allow_off_by_one=True),
'lwk_off_by_one': make_scorer(kappa, weights='linear',
allow_off_by_one=True),
'uwk_off_by_one': make_scorer(kappa, allow_off_by_one=True)}
SCORERS.update(_scorers)
_scorers = {
'f1_score_micro':
make_scorer(f1_score, average='micro', pos_label=None),
'f1_score_macro':
make_scorer(f1_score, average='macro', pos_label=None),
'f1_score_weighted':
make_scorer(f1_score, average='weighted', pos_label=None),
'f1_score_least_frequent':
make_scorer(f1_score_least_frequent),
'pearson':
make_scorer(pearson),
'spearman':
make_scorer(spearman),
'kendall_tau':
make_scorer(kendall_tau),
'unweighted_kappa':
make_scorer(kappa),
'quadratic_weighted_kappa':
make_scorer(kappa, weights='quadratic'),
'linear_weighted_kappa':
make_scorer(kappa, weights='linear'),
'qwk_off_by_one':
make_scorer(kappa, weights='quadratic', allow_off_by_one=True),
'lwk_off_by_one':
make_scorer(kappa, weights='linear', allow_off_by_one=True),
'uwk_off_by_one':
make_scorer(kappa, allow_off_by_one=True)
}
SCORERS.update(_scorers)
def test_scorer_memmap_input():
# Non-regression test for #6147: some score functions would
# return singleton memmap when computed on memmap data instead of scalar
# float values.
for name in SCORERS.keys():
yield check_scorer_memmap, name
def test_all_scorers_repr():
# Test that all scorers have a working repr
for name, scorer in SCORERS.items():
repr(scorer)
rmse = math.sqrt(mse)
print(rmse)
'''summary table for coefficients'''
import statsmodels.api as sm
from sklearn.linear_model import LinearRegression as lr
X2_train = sm.add_constant(X_train) # add a column of 1 beside x col
ols = sm.OLS(
y_train.astype(float),
X2_train.astype(float)) # ordinary least square = linear regression
lr = ols.fit()
print(lr.summary())
'''Cross validation'''
from sklearn.model_selection import KFold, cross_val_score
from sklearn.linear_model import LinearRegression as lr
from sklearn.metrics import SCORERS
SCORERS.keys()
kf = KFold(n_splits=5, shuffle=True, random_state=1)
lr = lr()
'''Cross validation score (R2 for test data, and full data)'''
r2score = cross_val_score(lr1, X_test, y_test, cv=kf, scoring='r2')
print(r2score.mean())
r2score_b = cross_val_score(lr, X, y, cv=kf, scoring='r2')
print(r2score_b.mean())
'''Cross validation score (RMSE for test data, and full data)'''
RMSE = np.sqrt(-cross_val_score(
lr1, X_test, y_test, cv=kf, scoring='neg_mean_squared_error'))
print(RMSE.mean())
RMSE_b = np.sqrt(
-cross_val_score(lr, X, y, cv=kf, scoring='neg_mean_squared_error'))
print(RMSE_b.mean())
X_test[sc_cols] = \
sc.transform(X_test[sc_cols])
features = list(X_train.columns.values)
print(X_train.shape)
print(X_test.shape)
##############################################################
param_grid = {
'C': [0.001, 0.01, 0.1, 1,
10], #[.1,.9], #[1e-5,1e-4, 1e-3, 1e-2, 0.1],# 1, 10, 100],
'gamma': [
0.001, 0.01, 0.1, 1
], #['scale',1,10,100], #'['auto','scale', 1e-5, 1e-4, 1e-3, 1e-2, 0.1, 1, 10, 100],
'kernel': ['sigmoid']
}
print(sorted(SCORERS.keys()))
clf = GridSearchCV(
svm.SVC(probability=True),
param_grid=param_grid,
scoring='accuracy', #'roc_auc', #'f1_macro',
# 'f1_weighted', #'precision_weighted',#'average_precision', #'f1_macro',
cv=3,
refit=True,
verbose=10, # 10 to see results
return_train_score=True,
# n_jobs=multiprocessing.cpu_count() - 5
# get error if using too much memory , njobs < cpu's availale -2
#n_jobs=30
) # higher verbose =more printed
import random
from pprint import pprint
from numpy import ravel
from sklearn.datasets import load_boston
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import accuracy_score, SCORERS
if __name__ == '__main__':
data = load_boston()
model = RandomForestRegressor(n_estimators=10, random_state=0)
cross = cross_val_score(model,
data.data,
data.target,
cv=10,
scoring='neg_mean_squared_error')
# 模型指标评估列表
k = sorted(SCORERS.keys())
pprint(k)
import pandas as pd
from sklearn.model_selection import cross_val_score
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import SCORERS
data = pd.read_csv('IRIS.csv')
print(data.describe())
print(data.head())
y = data['species']
X = data.drop(['species'], axis=1)
print(y.head())
print(X.head())
dt_model = DecisionTreeClassifier()
print(SCORERS.keys())
scores = cross_val_score(dt_model, X, y, cv=5, n_jobs=4, scoring='accuracy')
print(scores)
print(scores.mean())
print(scores.std())
'neg_root_mean_squared_error': 'neg_root_mean_squared_error',
'neg_mean_poisson_deviance': 'neg_mean_poisson_deviance',
'neg_mean_gamma_deviance': 'neg_mean_gamma_deviance'
},
}
AVALIABLE['categorical'] = AVALIABLE['binary'].copy()
# Set defaults
AVALIABLE['binary']['default'] = 'roc_auc'
AVALIABLE['regression']['default'] = 'r2'
AVALIABLE['categorical']['default'] = 'roc_auc_ovr'
SCORERS.update({
'neg_hamming':
M.make_scorer(score_func=M.hamming_loss,
greater_is_better=False,
needs_threshold=False),
'matthews':
M.make_scorer(score_func=M.matthews_corrcoef, greater_is_better=True)
})
def get_scorer_from_str(scorer_str):
return SCORERS[scorer_str]
def get_scorers_by_type(problem_type):
objs = []
for scorer_str in AVALIABLE[problem_type]:
conv = AVALIABLE[problem_type][scorer_str]
"""
Appends new scorers in to the SCORERS constant.
"""
from sklearn.metrics import SCORERS
from sklearn.metrics import make_scorer
from imblearn.metrics import geometric_mean_score
from ..metrics.classification import tp_score, tn_score, fp_score, fn_score
from ..metrics.regression import weighted_mean_squared_error
__all__ = [
'geometric_mean_score', 'tp_score', 'tn_score', 'fp_score', 'fn_score',
'weighted_mean_squared_error', 'SCORERS'
]
APPENDED_SCORERS = {
'tp': (tp_score, ),
'tn': (tn_score, ),
'fp': (fp_score, False),
'fn': (fn_score, False),
'geometric_mean_score': (geometric_mean_score, ),
'weighted_mean_squared_error': (weighted_mean_squared_error, )
}
APPENDED_SCORERS = {
name: make_scorer(*sc_func)
for name, sc_func in APPENDED_SCORERS.items()
}
SCORERS.update(APPENDED_SCORERS)
def get_scorer_name(scorer):
"""Return the name of the provided scorer."""
for key, value in SCORERS.items():
if scorer.__dict__ == value.__dict__:
return key
def evaluate(self, params, df):
"""Evaluates the data.
Evaluates the data with a given scoring function and given hyper-parameters
of the whole pipeline. If no parameters are set, default configuration for
each step is evaluated : no feature selection is applied and no meta features are
created.
Parameters
----------
params : dict, default = None.
Hyper-parameters dictionary for the whole pipeline.
- The keys must respect the following syntax : "enc__param".
- "enc" = "ne" for na encoder
- "enc" = "ce" for categorical encoder
- "enc" = "fs" for feature selector [OPTIONAL]
- "enc" = "stck"+str(i) to add layer n°i of meta-features [OPTIONAL]
- "enc" = "est" for the final estimator
- "param" : a correct associated parameter for each step. Ex: "max_depth" for "enc"="est", ...
- The values are those of the parameters. Ex: 4 for key = "est__max_depth", ...
df : dict, default = None
Dataset dictionary. Must contain keys and values:
- "train": pandas DataFrame for the train set.
- "target" : encoded pandas Serie for the target on train set (with dtype='float' for a regression or dtype='int' for a classification). Indexes should match the train set.
Returns
-------
float.
The score. The higher the better.
Positive for a score and negative for a loss.
Examples
--------
>>> from mlbox.optimisation import *
>>> from sklearn.datasets import load_boston
>>> #load data
>>> dataset = load_boston()
>>> #evaluating the pipeline
>>> opt = Optimiser()
>>> params = {
... "ne__numerical_strategy" : 0,
... "ce__strategy" : "label_encoding",
... "fs__threshold" : 0.1,
... "stck__base_estimators" : [Regressor(strategy="RandomForest"), Regressor(strategy="ExtraTrees")],
... "est__strategy" : "Linear"
... }
>>> df = {"train" : pd.DataFrame(dataset.data), "target" : pd.Series(dataset.target)}
>>> opt.evaluate(params, df)
"""
ne = NA_encoder()
ce = Categorical_encoder()
##########################################
# Automatically checking the task
##########################################
# TODO: a lot of code can be factorized for the different tasks
##########################################
# Classification
##########################################
if (df['target'].dtype == 'int'):
# Cross validation
counts = df['target'].value_counts()
classes_to_drop = counts[counts < self.n_folds].index
mask_to_drop = df['target'].apply(lambda x: x in classes_to_drop)
indexes_to_drop = df['target'][mask_to_drop].index
n_classes = len(counts) - len(classes_to_drop)
if n_classes == 1:
raise ValueError(
"Your target has not enough classes. You can't run the optimiser"
)
cv = StratifiedKFold(n_splits=self.n_folds,
shuffle=True,
random_state=self.random_state)
# Estimator
est = Classifier()
# Feature selection if specified
fs = None
if (params is not None):
for p in params.keys():
if (p.startswith("fs__")):
fs = Clf_feature_selector()
else:
pass
# Stacking if specified
STCK = {}
if (params is not None):
for p in params.keys():
if (p.startswith("stck")):
# TODO: Check if p.split("__")[1] instead?
STCK[p.split("__")[0]] = StackingClassifier(
verbose=False) # noqa
else:
pass
# Default scoring for classification
if (self.scoring is None):
self.scoring = 'neg_log_loss' # works also for multiclass pb
else:
if (type(self.scoring) == str):
if (self.scoring not in list(SCORERS.keys())):
warnings.warn("Unknown or invalid scoring metric. "
"neg_log_loss is used instead.")
self.scoring = 'neg_log_loss'
else:
# binary classification
if n_classes <= 2:
pass
# multiclass classification
else:
warnings.warn(
"This is a multiclass problem. Please make sure that your scoring metric is "
"appropriate.")
if self.scoring + "_weighted" in list(
SCORERS.keys()):
warnings.warn(
"Weighted strategy for the scoring metric is used."
)
self.scoring = self.scoring + "_weighted"
# specific scenarios
else:
if self.scoring == "roc_auc":
self.scoring = make_scorer(
lambda y_true, y_pred: roc_auc_score(
pd.get_dummies(y_true), y_pred
), # noqa
greater_is_better=True,
needs_proba=True)
else:
pass
##########################################
# Regression
##########################################
elif (df['target'].dtype == 'float'):
# Cross validation
indexes_to_drop = []
cv = KFold(n_splits=self.n_folds,
shuffle=True,
random_state=self.random_state)
# Estimator
est = Regressor()
# Feature selection if specified
fs = None
if (params is not None):
for p in params.keys():
if (p.startswith("fs__")):
fs = Reg_feature_selector()
else:
pass
# Stacking if specified
STCK = {}
if (params is not None):
for p in params.keys():
if (p.startswith("stck")):
# TODO: Check if p.split("__")[1] instead?
STCK[p.split("__")[0]] = StackingRegressor(
verbose=False)
else:
pass
# Default scoring for regression
if (self.scoring is None):
self.scoring = "neg_mean_squared_error"
else:
if (type(self.scoring) == str):
if (self.scoring not in list(SCORERS.keys())):
warnings.warn(
"Unknown or invalid scoring metric. "
"neg_mean_squared_error is used instead.")
self.scoring = 'neg_mean_squared_error'
else:
pass
else:
pass
else:
raise ValueError("Impossible to determine the task. "
"Please check that your target is encoded.")
##########################################
# Creating the Pipeline
##########################################
pipe = [("ne", ne), ("ce", ce)]
# Do we need to cache transformers?
cache = False
if (params is not None):
if ("ce__strategy" in params):
if (params["ce__strategy"] == "entity_embedding"):
cache = True
else:
pass
else:
pass
if (fs is not None):
if ("fs__strategy" in params):
if (params["fs__strategy"] != "variance"):
cache = True
else:
pass
else:
pass
if (len(STCK) != 0):
cache = True
else:
pass
# Pipeline creation
if (fs is not None):
pipe.append(("fs", fs))
else:
pass
for stck in np.sort(list(STCK)):
pipe.append((stck, STCK[stck]))
pipe.append(("est", est))
if cache:
pp = Pipeline(pipe, memory=self.to_path)
else:
pp = Pipeline(pipe)
##########################################
# Fitting the Pipeline
##########################################
start_time = time.time()
# No params : default configuration
if (params is None):
set_params = True
print('No parameters set. Default configuration is tested')
else:
try:
pp = pp.set_params(**params)
set_params = True
except:
set_params = False
if (set_params):
if (self.verbose):
print("")
print("#####################################################"
" testing hyper-parameters... "
"#####################################################")
print("")
print(">>> NA ENCODER :" + str(ne.get_params()))
print("")
print(">>> CA ENCODER :" + str({'strategy': ce.strategy}))
if (fs is not None):
print("")
print(">>> FEATURE SELECTOR :" + str(fs.get_params()))
for i, stck in enumerate(np.sort(list(STCK))):
stck_params = STCK[stck].get_params().copy()
stck_params_display = {
k: stck_params[k]
for k in stck_params.keys() if k not in
["level_estimator", "verbose", "base_estimators"]
}
print("")
print(">>> STACKING LAYER n°" + str(i + 1) + " :" +
str(stck_params_display))
for j, model in enumerate(stck_params["base_estimators"]):
print("")
print(" > base_estimator n°" + str(j + 1) + " :" +
str(
dict(
list(model.get_params().items()) +
list(model.get_estimator().get_params().
items()))))
print("")
print(">>> ESTIMATOR :" + str(
dict(
list(est.get_params().items()) +
list(est.get_estimator().get_params().items()))))
print("")
try:
# Computing the mean cross validation score across the folds
scores = cross_val_score(estimator=pp,
X=df['train'].drop(indexes_to_drop),
y=df['target'].drop(indexes_to_drop),
scoring=self.scoring,
cv=cv)
score = np.mean(scores)
except:
scores = [-np.inf for _ in range(self.n_folds)]
score = -np.inf
else:
raise ValueError("Pipeline cannot be set with these parameters."
" Check the name of your stages.")
if (score == -np.inf):
warnings.warn(
"An error occurred while computing the cross "
"validation mean score. Please check that the parameter values are correct "
"and that your scoring function is valid and appropriate to the task."
)
##########################################
# Reporting scores
##########################################
out = " ("
for i, s in enumerate(scores[:-1]):
out = out + "fold " + str(i + 1) + " = " + str(s) + ", "
if (self.verbose):
print("")
print("MEAN SCORE : " + str(self.scoring) + " = " + str(score))
print("VARIANCE : " + str(np.std(scores)) + out + "fold " +
str(i + 2) + " = " + str(scores[-1]) + ")")
print("CPU time: %s seconds" % (time.time() - start_time))
print("")
return score
from sklearn.metrics import SCORERS
if __name__ == "__main__":
print("These below are going to be lots of fun")
for scorer in SCORERS.keys():
print(scorer)
import xgboost as xgb
from xgboost import XGBRegressor as XGBR
from xgboost import XGBClassifier as XGBC
from sklearn.datasets import load_boston
from sklearn.model_selection import train_test_split, KFold, cross_val_score
from sklearn.ensemble import RandomForestClassifier as RFC
from sklearn.linear_model import LinearRegression as LR
from sklearn.metrics import mean_squared_error as mse, SCORERS
data = load_boston()
X = data.data
Y = data.target
X_trian, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.3)
sk_xgb_model = XGBR(n_estimators=100, random_state=0).fit(X_trian, Y_train)
pre1 = sk_xgb_model.predict(X_test)
score1 = sk_xgb_model.score(X_test, Y_test)
mse = mse(y_true=Y_test, y_pred=pre1)
important = sk_xgb_model.feature_importances_
print('pre: ', pre1)
print('score1: ', score1)
print('mse: ', mse)
print('important: ', important)
print('mean: ', Y.mean())
print(SCORERS.keys()) #所有可用的评估指标
def test_all_scorers_repr():
# Test that all scorers have a working repr
for name, scorer in SCORERS.items():
repr(scorer)
['grid_search', 'random_search', 'bayesian_search']),
required=True)
@click.option(
'--cv',
help="Number of cross validation steps",
type=int,
required=False,
default=5,
show_default=True,
)
@click.option(
'-m',
'--metrics',
help=
"Metrics that should be tested during cross validation (comma separated)",
type=click.Choice(list(SCORERS.keys())),
required=False,
multiple=True,
)
@click.option(
'--randomize',
help=
"Randomize sample labels to test the stability of and effectiveness of the machine learning algorithm",
is_flag=True,
required=False,
)
def classify(
data: str,
out: str,
model: str,
optimizer: str,
