def _evalFunction(individual, searchobj, name_values, X, y, scorer, cv, iid, fit_params,
verbose=0, error_score='raise'):
parameters = _individual_to_params(individual, name_values)
score = 0
n_test = 0
for train, test in cv:
paramkey = str(parameters)
if paramkey in searchobj.score_cache:
searchobj.num_cache_hits += 1
_score = searchobj.score_cache[paramkey]
else:
_score, _, _ = _fit_and_score(estimator=individual.est, X=X, y=y, scorer=scorer,
train=train, test=test, verbose=verbose,
parameters=parameters, fit_params=fit_params,
error_score=error_score)
searchobj.num_evaluations += 1
searchobj.score_cache[paramkey] = _score
if searchobj.verbose and (searchobj.num_evaluations + searchobj.num_cache_hits) % searchobj.population_size == 0:
print "Scoring evaluations: %d, Cache hits: %d, Total: %d" % (
searchobj.num_evaluations, searchobj.num_cache_hits, searchobj.num_evaluations + searchobj.num_cache_hits)
if iid:
score += _score*len(test)
n_test += len(test)
else:
score += _score
n_test += 1
score /= float(n_test)
return (score,)
def _evalFunction(individual,
name_values,
X,
y,
scorer,
cv,
iid,
fit_params,
verbose=0,
error_score='raise',
score_cache={}):
""" Developer Note:
--------------------
score_cache was purposefully moved to parameters, and given a dict reference.
It will be modified in-place by _evalFunction based on it's reference.
This is to allow for a managed, paralell memoization dict,
and also for different memoization per instance of EvolutionaryAlgorithmSearchCV.
Remember that dicts created inside function definitions are presistent between calls,
So unless it is replaced this function will be memoized each call automatically. """
parameters = _individual_to_params(individual, name_values)
score = 0
n_test = 0
paramkey = str(individual)
if paramkey in score_cache:
score = score_cache[paramkey]
else:
for train, test in cv.split(X, y):
assert len(train) > 0 and len(
test
) > 0, "Training and/or testing not long enough for evaluation."
try:
_score = _fit_and_score(estimator=individual.est,
X=X,
y=y,
scorer=scorer,
train=train,
test=test,
verbose=verbose,
parameters=parameters,
fit_params=fit_params,
error_score=error_score)[0]
except:
return (-np.inf, )
if iid:
score += _score * len(test)
n_test += len(test)
else:
score += _score
n_test += 1
assert n_test > 0, "No fitting was accomplished, check data and cross validation method."
score /= float(n_test)
score_cache[paramkey] = score
return (score, )
def evaluate(self, dataset, pipelines):
if not self.is_valid(dataset):
raise AssertionError("Dataset is not appropriate for evaluation")
for subject in dataset.subject_list:
# check if we already have result for this subject/pipeline
# we might need a better granularity, if we query the DB
run_pipes = self.results.not_yet_computed(pipelines, dataset, subject)
if len(run_pipes) == 0:
continue
# get the data
X, y, metadata = self.paradigm.get_data(
dataset, [subject], self.return_epochs
)
le = LabelEncoder()
y = y if self.mne_labels else le.fit_transform(y)
groups = metadata.session.values
scorer = get_scorer(self.paradigm.scoring)
for name, clf in run_pipes.items():
# we want to store a results per session
cv = LeaveOneGroupOut()
for train, test in cv.split(X, y, groups):
t_start = time()
if isinstance(X, BaseEpochs):
cvclf = clone(clf)
cvclf.fit(X[train], y[train])
score = scorer(cvclf, X[test], y[test])
else:
result = _fit_and_score(
clone(clf),
X,
y,
scorer,
train,
test,
verbose=False,
parameters=None,
fit_params=None,
error_score=self.error_score,
)
score = result["test_scores"]
duration = time() - t_start
nchan = X.info["nchan"] if isinstance(X, BaseEpochs) else X.shape[1]
res = {
"time": duration,
"dataset": dataset,
"subject": subject,
"session": groups[test][0],
"score": score,
"n_samples": len(train),
"n_channels": nchan,
"pipeline": name,
}
yield res
def _evalFunction(individual,
name_values,
X,
y,
scorer,
cv,
uniform,
fit_params,
verbose=0,
error_score='raise',
score_cache={}):
"""[Evaluación del modelo]
Arguments:
individual {[creator.Individual]} -- [Individuo]
name_values {[list]} -- [parámetros en general]
X {[array]} -- [Input]
y {[array]} -- [Output]
scorer {[string]} -- [Parámetro de evaluación, precisión]
cv {[int | cross-validation]} -- [Especificación de los folds]
uniform {[boolean]} -- [True hace que la data se distribuya uniformemente en los folds]
fit_params {[dict | None]} -- [parámetros para estimator.fit]
Keyword Arguments:
verbose {integer} -- [Mensajes de descripción] (default: {0})
error_score {numerico} -- [valor asignado si ocurre un error en fitting] (default: {'raise'})
score_cache {dict} -- [description] (default: {{}})
"""
parameters = _individual_to_params(individual, name_values)
score = 0
n_test = 0
paramkey = str(individual)
if paramkey in score_cache:
score = score_cache[paramkey]
else:
for train, test in cv.split(X, y):
_score = _fit_and_score(estimator=individual.est,
X=X,
y=y,
scorer=scorer,
train=train,
test=test,
verbose=verbose,
parameters=parameters,
fit_params=fit_params,
error_score=error_score)[0]
if uniform:
score += _score * len(test)
n_test += len(test)
else:
score += _score
n_test += 1
assert n_test > 0, "No se completo el fitting, Verificar data."
score /= float(n_test)
score_cache[paramkey] = score
return (score, )
def _wrapped_cross_val_score(sklearn_pipeline, features, target,
cv, scoring_function, sample_weight=None, groups=None):
"""Fit estimator and compute scores for a given dataset split.
Parameters
----------
sklearn_pipeline : pipeline object implementing 'fit'
The object to use to fit the data.
features : array-like of shape at least 2D
The data to fit.
target : array-like, optional, default: None
The target variable to try to predict in the case of
supervised learning.
cv: int or cross-validation generator
If CV is a number, then it is the number of folds to evaluate each
pipeline over in k-fold cross-validation during the TPOT optimization
process. If it is an object then it is an object to be used as a
cross-validation generator.
scoring_function : callable
A scorer callable object / function with signature
``scorer(estimator, X, y)``.
sample_weight : array-like, optional
List of sample weights to balance (or un-balanace) the dataset target as needed
groups: array-like {n_samples, }, optional
Group labels for the samples used while splitting the dataset into train/test set
"""
sample_weight_dict = set_sample_weight(sklearn_pipeline.steps, sample_weight)
features, target, groups = indexable(features, target, groups)
cv = check_cv(cv, target, classifier=is_classifier(sklearn_pipeline))
cv_iter = list(cv.split(features, target, groups))
scorer = check_scoring(sklearn_pipeline, scoring=scoring_function)
try:
with warnings.catch_warnings():
warnings.simplefilter('ignore')
scores = [_fit_and_score(estimator=clone(sklearn_pipeline),
X=features,
y=target,
scorer=scorer,
train=train,
test=test,
verbose=0,
parameters=None,
fit_params=sample_weight_dict)
for train, test in cv_iter]
CV_score = np.array(scores)[:, 0]
return np.nanmean(CV_score)
except TimeoutException:
return "Timeout"
except Exception as e:
return -float('inf')
def fun(tup):
# DO NOT REFERENCE TO `self` ANYWHERE IN THIS FUNCTION.
# IT WILL CAUSE A SPARK-5063 ERROR.
(index, (parameters, train, test)) = tup
local_estimator = clone(base_estimator)
local_X = X_bc.value
local_y = y_bc.value
res = _fit_and_score(local_estimator, local_X, local_y, scorers,
train, test, verbose, parameters, fit_params=fit_params,
return_train_score=return_train_score,
return_n_test_samples=True, return_times=True,
error_score=error_score)
return (index, res)
def _evalFunction(individual, name_values, X, y, scorer, cv, uniform, fit_params,
verbose=0, error_score='raise', score_cache={}, result_cache=[]):
parameters = _individual_to_params(individual, name_values)
nombreModelo = str(individual.est).split('(')[0] # individual.est.__class__.__name__
score = 0
paramkey = nombreModelo+str(individual)
if 'genCount' in score_cache:
score_cache['genCount'] = score_cache['genCount'] + 1
else:
score_cache['genCount'] = 1
if paramkey in score_cache:
score = score_cache[paramkey]
else:
try:
resultIndividuo = []
scorer = { 'mae': make_scorer(mae), 'mse': make_scorer(mse), 'approach': make_scorer(distance2d) }
for train, test in cv.split(X, y):
resultIndividuo.append(_fit_and_score(estimator=individual.est, X=X, y=y, scorer=scorer, parameters=parameters,
train=train, test=test, verbose=verbose, fit_params=None, return_times=True))
df = pd.DataFrame(list(map(lambda x: _evalfs(x), resultIndividuo)))
accuracy = np.array(resultIndividuo)[:, 0] # accuracy
runtime = np.array(resultIndividuo)[:, 2] + np.array(resultIndividuo)[:, 1] # runtime train+test
score = df['approach'].mean()
score_cache[paramkey] = score
dict_result = parameters
dict_result['Accuracy'] = score
dict_result['stdApproach'] = df['approach'].std()
dict_result['MSE'] = df['mse'].mean()
dict_result['stdMSE'] = df['mse'].std()
dict_result['MAE'] = df['mae'].mean()
dict_result['stdMAE'] = df['mae'].std()
dict_result['Runtime'] = df['time'].mean()
dict_result['stdRuntime'] = df['time'].std()
dict_result['genCount'] = score_cache['genCount']
result_cache.append(dict_result)
except Exception as ex:
print(ex)
score_cache[paramkey] = 0
dict_result = parameters
dict_result['Accuracy'] = 0
dict_result['stdApproach'] = 0
dict_result['Runtime'] = 0
dict_result['stdRuntime'] = 0
dict_result['MSE'] = 100
dict_result['stdMSE'] = 100
dict_result['MAE'] = 100
dict_result['stdMAE'] = 100
dict_result['genCount'] = score_cache['genCount']
result_cache.append(dict_result)
return (score,)
def getModelAccuracy(self, parametros, individual, score_cache, resultados,
generacion):
params = _individual_to_params(individual, parametros)
score = 0
scoring = "accuracy"
nombreModelo = str(self.estimator).split('(')[0]
paramkey = nombreModelo + str(np.int32(individual))
if paramkey in score_cache:
score = score_cache[paramkey]
else:
try:
resultIndividuo = []
cv = KFold(n_splits=10, shuffle=False)
scorer = check_scoring(self.estimator, scoring=scoring)
for train, test in cv.split(self.X, self.y):
resultIndividuo.append(
_fit_and_score(estimator=self.estimator,
X=self.X,
y=self.y,
scorer=scorer,
parameters=params,
train=train,
test=test,
verbose=0,
fit_params=None,
return_times=True))
accuracy = np.array(resultIndividuo)[:, 0] # accuracy
runtime = np.array(resultIndividuo)[:, 2] + np.array(
resultIndividuo)[:, 1] # runtime train+test
score = accuracy.mean()
score_cache[paramkey] = score
dict_result = params
dict_result['Accuracy'] = score
dict_result['stdAccuracy'] = accuracy.std()
dict_result['Runtime'] = runtime.mean()
dict_result['stdRuntime'] = runtime.std()
dict_result['generacion'] = generacion
resultados.append(dict_result)
except Exception as ex:
print(ex)
score_cache[paramkey] = 0
dict_result = params
dict_result['Accuracy'] = 0
dict_result['stdAccuracy'] = 0
dict_result['Runtime'] = 0
dict_result['stdRuntime'] = 0
dict_result['generacion'] = generacion
resultados.append(dict_result)
return score
def evaluate_candidates(candidate_params):
if isinstance(candidate_params, dict) or isinstance(
candidate_params, defaultdict):
candidate_params = list(candidate_params)
n_candidates = len(candidate_params)
if self.verbose > 0:
print(
"Fitting {0} folds for each of {1} remaining candidates, totalling {2} fits"
.format(n_splits, n_candidates, n_candidates * n_splits))
# print('list(cv.split(X, y, groups)): %s' % list(cv.split(X, y, groups)))
# print('list(product(candidate_params, cv.split(X, y, groups))): %s' % list(product(candidate_params, cv.split(X, y, groups))))
fold_num = 0
for parameters, (train, test) in product(candidate_params,
cv.split(X, y, groups)):
print('product index/fold number: %d' % fold_num)
# print('\tparams: %s' % parameters)
# print('\ttrain: %s' % train)
# print('\ttest: %s' % test)
out = _fit_and_score(estimator=clone(base_estimator),
X=X,
y=y,
train=train,
test=test,
parameters=parameters,
**fit_and_score_kwargs)
print('\tout: %s' % out)
all_candidate_params.extend(candidate_params)
all_out.extend(out)
# nonlocal keyword is exactly what it sounds like, uses the outer function scope: w3schools.com/python/ref_keyword_nonlocal.asp
nonlocal results
# results = self._format_results(all_candidate_params, scorers, n_splits, all_out)
result = self._format_result(candidate_param=parameters,
scorer=scorers,
n_splits=n_splits,
out=out)
results.append(result)
self.cv_results.append(result)
# Just finished training a model, should cv_results be saved?
if fold_num % self.cv_results_save_freq == 0:
self._save_cv_results()
fold_num += 1
return self.cv_results
def _evalFunction(individual,
name_values,
X,
y,
scorer,
cv,
iid,
fit_params,
verbose=0,
error_score='raise',
score_cache={}):
parameters = _individual_to_params(individual, name_values)
score = 0
n_test = 0
paramkey = str(individual)
if paramkey in score_cache:
score = score_cache[paramkey]
else:
for train, test in cv.split(X, y):
assert len(train) > 0 and len(
test
) > 0, "Training and/or testing not long enough for evaluation."
_score = _fit_and_score(estimator=individual.est,
X=X,
y=y,
scorer=scorer,
train=train,
test=test,
verbose=verbose,
parameters=parameters,
fit_params=fit_params,
error_score=error_score)[0]
if iid:
score += _score * len(test)
n_test += len(test)
else:
score += _score
n_test += 1
assert n_test > 0, "No fitting was accomplished, check data and cross validation method."
score /= float(n_test)
score_cache[paramkey] = score
return (score, )
def obj(x):
cand_params = {}
_idx = 0
for _param in self.params_list:
_param_num_range = self.params[_param]
if _param_num_range.VType != 'hdreal':
if _param_num_range.VType == 'integer':
cand_params[_param] = int(round(x[_idx]))
elif _param_num_range.VType == 'categorical':
cand_params[_param] = _param_num_range.items[int(round(x[_idx]))]
else:
cand_params[_param] = x[_idx]
_idx += 1
else:
_cls_dict = {}
for i_ in range(_param_num_range.n):
_cls_dict[target_classes[i_]] = x[_idx]
_idx += 1
cand_params[_param] = _cls_dict
cl = clone(self.estimator)
cl.set_params(**cand_params)
score = 0
n_test = 0
for train, test in cv_dat:
#if True:
try:
_score = _fit_and_score(estimator=cl, X=X, y=y, scorer=self.scorer_,
train=train, test=test, verbose=self.verbose,
parameters=cand_params, fit_params=self.fit_params,
error_score=self.error_score)[0]
if self.iid:
score += _score * len(test)
n_test += len(test)
else:
score += _score
n_test += 1
#else:
except ValueError:
pass
except:# LightGBMError:
pass
score /= float(max(n_test, 1))
if is_classifier(self.estimator):
return - score
else:
return score
def getModelAccuracy(parametros, individual, estimator, score_cache,
resultados):
X, y = _createDataset(
"Tx_0x06"
) # adaptado de búsqueda de configuración hay que adaptar para sacarlo
params = _individual_to_params(individual, parametros)
score = 0
scoring = "accuracy"
nombreModelo = str(estimator).split('(')[0]
paramkey = nombreModelo + str(np.int32(individual))
if paramkey in score_cache:
score = score_cache[paramkey]
else:
resultIndividuo = []
cv = KFold(n_splits=10, shuffle=False)
scorer = check_scoring(estimator, scoring=scoring)
for train, test in cv.split(X, y):
resultIndividuo.append(
_fit_and_score(estimator=estimator,
X=X,
y=y,
scorer=scorer,
parameters=params,
train=train,
test=test,
verbose=0,
fit_params=None,
return_times=True))
accuracy = np.array(resultIndividuo)[:, 0] #accuracy
runtime = np.array(resultIndividuo)[:, 2] + np.array(
resultIndividuo)[:, 1] #runtime train+test
# error = distance_error(estimator, X, y)
score = accuracy.mean()
score_cache[paramkey] = score
dict_result = {
'Modelo': nombreModelo,
'Parametros': params,
'Accuracy': accuracy.mean(),
'stdAccuracy': accuracy.std(),
'Runtime': runtime.mean(),
'accuracy_values': accuracy,
'runtime_values': runtime,
}
resultados.append(dict_result)
return score
def getModelApproach(self, parametros, individual, score_cache, resultados, generacion):
params = _individual_to_params(individual, parametros)
score = 0
scoring = "mse"
nombreModelo = str(self.estimator).split('(')[0]
paramkey = nombreModelo + str(np.int32(individual))
if paramkey in score_cache:
score = score_cache[paramkey]
else:
try:
resultIndividuo = []
cv = KFold(n_splits=10, shuffle=True, random_state=self.seed)
scorer = scoring_reg = { 'mae': make_scorer(mae), 'mse': make_scorer(mse), 'approach': make_scorer(distance2d) }
for train, test in cv.split(self.X, self.y):
resultIndividuo.append(_fit_and_score(estimator=self.estimator, X=self.X, y=self.y, scorer=scorer, parameters=params,
train=train, test=test, verbose=0, fit_params=None, return_times=True))
df = pd.DataFrame(list(map(lambda x: _evalfs(x), resultIndividuo)))
score = df['approach'].mean()
score_cache[paramkey] = score
dict_result = params
dict_result['Accuracy'] = score
dict_result['stdApproach'] = df['approach'].std()
dict_result['MSE'] = df['mse'].mean()
dict_result['stdMSE'] = df['mse'].std()
dict_result['MAE'] = df['mae'].mean()
dict_result['stdMAE'] = df['mae'].std()
dict_result['Runtime'] = df['time'].mean()
dict_result['stdRuntime'] = df['time'].std()
dict_result['generacion'] = generacion
resultados.append(dict_result)
except Exception as ex:
print(ex)
score_cache[paramkey] = 0
dict_result = params
dict_result['Accuracy'] = 0
dict_result['stdApproach'] = 0
dict_result['Runtime'] = 0
dict_result['stdRuntime'] = 0
dict_result['MSE'] = 100
dict_result['stdMSE'] = 100
dict_result['MAE'] = 100
dict_result['stdMAE'] = 100
dict_result['generacion'] = generacion
resultados.append(dict_result)
return score
def evaluate(self, dataset, pipelines):
if not self.is_valid(dataset):
raise AssertionError('Dataset is not appropriate for evaluation')
for subject in dataset.subject_list:
# check if we already have result for this subject/pipeline
# we might need a better granularity, if we query the DB
run_pipes = self.results.not_yet_computed(pipelines, dataset,
subject)
if len(run_pipes) == 0:
continue
# get the data
X, y, metadata = self.paradigm.get_data(dataset, [subject])
le = LabelEncoder()
y = le.fit_transform(y)
groups = metadata.session.values
scorer = get_scorer(self.paradigm.scoring)
for name, clf in run_pipes.items():
# we want to store a results per session
cv = LeaveOneGroupOut()
for train, test in cv.split(X, y, groups):
t_start = time()
score = _fit_and_score(clone(clf),
X,
y,
scorer,
train,
test,
verbose=False,
parameters=None,
fit_params=None)[0]
duration = time() - t_start
res = {
'time': duration,
'dataset': dataset,
'subject': subject,
'session': groups[test][0],
'score': score,
'n_samples': len(train),
'n_channels': X.shape[1],
'pipeline': name
}
yield res
def _evalFunction(individual, gaobject, estimator, X, y, cv, scorer, verbose, fit_params, caching):
individual_sum = np.sum(individual, axis=0)
if individual_sum == 0:
return -10000, individual_sum
individual_tuple = tuple(individual)
if caching and individual_tuple in gaobject.scores_cache:
return gaobject.scores_cache[individual_tuple], individual_sum
X_selected = X[:, np.array(individual, dtype=np.bool)]
scores = []
for train, test in cv.split(X, y):
score = _fit_and_score(estimator=estimator, X=X_selected, y=y, scorer=scorer,
train=train, test=test, verbose=verbose, parameters=None,
fit_params=fit_params)
scores.append(score)
scores_mean = np.mean(scores)
if caching:
gaobject.scores_cache[individual_tuple] = scores_mean
return scores_mean, individual_sum
def spark_task(tup):
(index, (parameters, train, test)) = tup
local_estimator = clone(base_estimator)
local_X = X_bc.value
local_y = y_bc.value
error = None
with warnings.catch_warnings(record=True) as warns:
try:
res = _fit_and_score(
local_estimator, X=local_X, y=local_y, train=train,
test=test, parameters=parameters,
**fit_and_score_kwargs)
except Exception as e:
res = None
error = e
return index, {"results": res, "errors": error, "warnings": warns}
def _fit_score_and_log(self,
estimator,
X,
y,
scorer,
train,
test,
verbose,
parameters,
fit_params,
return_train_score=False,
return_parameters=False,
return_n_test_samples=False,
return_times=False,
return_estimator=False,
error_score=np.nan):
fit_result = _fit_and_score(estimator, X, y, scorer, train, test,
verbose, parameters, fit_params, True,
True, True, True, True, error_score)
test_score = fit_result[0]['score']
train_score = fit_result[1]['score']
sample_count = fit_result[2]
durations = [fit_result[3], fit_result[4]]
parameters = fit_result[5]
estimator = fit_result[6]
self.log_results(train_score, test_score, sample_count, durations,
parameters, estimator)
if return_train_score:
ret = [fit_result[0], fit_result[1]]
else:
ret = [fit_result[0]]
if return_n_test_samples:
ret.append(sample_count)
if return_times:
ret.extend(durations)
if return_parameters:
ret.append(parameters)
if return_estimator:
ret.append(estimator)
return ret
def test_one_parameter(task):
(index, (parameters, split_idx)) = task
local_estimator = clone(base_estimator)
local_x = x_bc.value
local_y = y_bc.value
local_groups = groups_bc.value
train, test = next(
islice(cv.split(local_x, local_y, local_groups), split_idx,
split_idx + 1))
res = _fit_and_score(local_estimator,
local_x,
local_y,
train=train,
test=test,
parameters=parameters,
**fit_and_score_kwargs)
return index, res
def grid_search_cv(model, param_grid, precomputed_kernels, y, cv=5):
cv = StratifiedKFold(n_splits=cv, shuffle=False)
results =[]
for train_index, test_index in cv.split(precomputed_kernels[0], y):
split_results = []
params = []
for idx, K in enumerate(precomputed_kernels):
for p in list(ParameterGrid(param_grid)):
sc = _fit_and_score(clone(model), K, y, scorer=make_scorer(accuracy_score),
train=train_index, test=test_index, verbose=0, parameters=p,fit_params=None)
split_results.append(sc)
params.append({'K_idx':idx, 'params':p})
results.append(split_results)
results=np.array(results)
fin_results = results.mean(axis=0)
best_idx = np.argmax(fin_results)
print(best_idx, fin_results[best_idx])
ret_model = clone(model).set_params(**params[best_idx]['params'])
return ret_model.fit(precomputed_kernels[params[best_idx]['K_idx']], y), params[best_idx]
def _evalFunction(individual, gaobject, estimator, X, y, cv, scorer, verbose, fit_params,
max_features, caching):
individual_sum = np.sum(individual, axis=0)
if individual_sum == 0 or individual_sum > max_features:
return -10000, individual_sum
individual_tuple = tuple(individual)
if caching and individual_tuple in gaobject.scores_cache:
return gaobject.scores_cache[individual_tuple], individual_sum
X_selected = X[:, np.array(individual, dtype=np.bool)]
scores = []
for train, test in cv.split(X, y):
score = _fit_and_score(estimator=estimator, X=X_selected, y=y, scorer=scorer,
train=train, test=test, verbose=verbose, parameters=None,
fit_params=fit_params)
scores.append(score)
scores_mean = np.mean(scores)
if caching:
gaobject.scores_cache[individual_tuple] = scores_mean
return scores_mean, individual_sum
def _evalFunctionClassifier(individual, name_values, X, y, scorer, cv, uniform, fit_params,
verbose=0, error_score='raise', score_cache={}, result_cache=[]):
parameters = _individual_to_params(individual, name_values)
nombreModelo = str(individual.est).split('(')[0]
score = 0
paramkey = nombreModelo+str(individual)
if 'genCount' in score_cache:
score_cache['genCount'] = score_cache['genCount'] + 1
else:
score_cache['genCount'] = 1
if paramkey in score_cache:
score = score_cache[paramkey]
else:
try:
resultIndividuo = []
scorer = check_scoring(individual.est, scoring="accuracy")
for train, test in cv.split(X, y):
resultIndividuo.append(_fit_and_score(estimator=individual.est, X=X, y=y, scorer=scorer,
train=train, test=test, verbose=verbose, parameters=parameters, fit_params=None, return_times=True))
accuracy = np.array(resultIndividuo)[:, 0] # accuracy
runtime = np.array(resultIndividuo)[:, 2] + np.array(resultIndividuo)[:, 1] # runtime train+test
score = accuracy.mean()
score_cache[paramkey] = score
dict_result = parameters
dict_result['Accuracy'] = score
dict_result['stdAccuracy'] = accuracy.std()
dict_result['Runtime'] = runtime.mean()
dict_result['stdRuntime'] = runtime.std()
dict_result['genCount'] = score_cache['genCount']
result_cache.append(dict_result)
except Exception as ex:
print(ex)
score_cache[paramkey] = 0
dict_result = parameters
dict_result['Accuracy'] = 0
dict_result['stdAccuracy'] = 0
dict_result['Runtime'] = 0
dict_result['stdRuntime'] = 0
dict_result['genCount'] = score_cache['genCount']
result_cache.append(dict_result)
return (score,)
def _evalFunction(individual,
name_values,
X,
y,
scorer,
cv,
uniform,
fit_params,
verbose=0,
error_score='raise',
score_cache={}):
parameters = _individual_to_params(individual, name_values)
score = 0
n_test = 0
paramkey = str(individual)
if paramkey in score_cache:
score = score_cache[paramkey]
else:
for train, test in cv.split(X, y):
_score = _fit_and_score(estimator=individual.est,
X=X,
y=y,
scorer=scorer,
train=train,
test=test,
verbose=verbose,
parameters=parameters,
fit_params=fit_params,
error_score=error_score)[0]
if uniform:
score += _score * len(test)
n_test += len(test)
else:
score += _score
n_test += 1
assert n_test > 0, "No se completó el fitting, verificar data."
score /= float(n_test)
score_cache[paramkey] = score
return (score, )
def fit_and_get_score(estimator, X, y):
cv = check_cv(cv=K_FOLD_CROSS_VALIDATION,
y=y,
classifier=is_classifier(estimator))
scorer = check_scoring(estimator, scoring=None)
scores = []
for train, test in cv.split(X, y):
score = _fit_and_score(estimator=estimator,
X=X,
y=y,
scorer=scorer,
train=train,
test=test,
verbose=0,
parameters=None,
fit_params=None)
scores.append(score)
scores_mean = np.mean(scores)
return scores_mean
def parallel_fit_score(cl, cand_params, X, y, scorer, train, test,
verbose, fit_params, error_score):
cl.set_params(**cand_params)
try:
_score = _fit_and_score(
estimator=cl,
X=X,
y=y,
scorer=scorer, #
train=train,
test=test,
verbose=verbose, #
parameters=cand_params,
fit_params=fit_params, #
error_score=error_score, #
)[0]
return _score
except ValueError:
pass
except: # LightGBMError:
pass
return None
def _evalFunction(individual, name_values, X, y, scorer, cv, uniform, fit_params,
verbose=0, error_score='raise', score_cache={}):
"""[Evaluación del modelo]
Arguments:
individual {[creator.Individual]} -- [Individuo]
name_values {[list]} -- [parámetros en general]
X {[array]} -- [Input]
y {[array]} -- [Output]
scorer {[string]} -- [Parámetro de evaluación, precisión]
cv {[int | cross-validation]} -- [Especificación de los folds]
uniform {[boolean]} -- [True hace que la data se distribuya uniformemente en los folds]
fit_params {[dict | None]} -- [parámetros para estimator.fit]
Keyword Arguments:
verbose {integer} -- [Mensajes de descripción] (default: {0})
error_score {numerico} -- [valor asignado si ocurre un error en fitting] (default: {'raise'})
score_cache {dict} -- [description] (default: {{}})
"""
parameters = _individual_to_params(individual, name_values)
score = 0
n_test = 0
paramkey = str(individual)
if paramkey in score_cache:
score = score_cache[paramkey]
else:
for train, test in cv.split(X, y):
_score = _fit_and_score(estimator=individual.est, X=X, y=y, scorer=scorer,
train=train, test=test, verbose=verbose,
parameters=parameters, fit_params=fit_params,
error_score=error_score)[0]
if uniform:
score += _score * len(test)
n_test += len(test)
else:
score += _score
n_test += 1
assert n_test > 0, "No se completo el fitting, Verificar data."
score /= float(n_test)
score_cache[paramkey] = score
return (score,)
def fit_grid_point(X,
y,
estimator,
parameters,
train,
test,
scorer,
verbose,
error_score=np.nan,
**fit_params):
check_scoring(estimator, scorer)
scores, n_samples_test = _fit_and_score(estimator,
X,
y,
scorer,
train,
test,
verbose,
parameters,
fit_params=fit_params,
return_n_test_samples=True,
error_score=error_score)
return scores, parameters, n_samples_test
def process_batch(self, work_batch):
fit_params = self.fit_params if self.fit_params is not None else {}
LOG.debug("Node %d received %d work items", comm_rank, len(work_batch))
results = []
for fold_id, train_index, test_index, parameters in work_batch:
ret = _fit_and_score(clone(self.estimator),
self._data_X, self._data_y,
self.scorer, train_index, test_index,
self.verbose, parameters, fit_params,
return_n_test_samples=True,
return_times=True)
result = parameters.copy()
result['score'] = ret[0]
result['n_samples_test'] = ret[1]
result['scoring_time'] = ret[2]
result['fold'] = fold_id
results.append(result)
LOG.debug("Node %d is done with fold %d", comm_rank, fold_id)
return results
def _evaluate_one(self, estimator, data_preproc, scorers):
res = []
for X_train, X_test, y_train, y_test in data_preproc:
X = np.vstack([X_train, X_test])
if y_train.ndim < 2 and y_test.ndim < 2:
y = np.hstack([y_train, y_test])
else:
y = np.vstack([y_train, y_test])
train = np.arange(len(X_train))
test = np.arange(len(X_train), len(X_test) + len(X_train))
with warnings.catch_warnings():
warnings.filterwarnings('ignore',
category=UndefinedMetricWarning)
test_scores = _fit_and_score(estimator,
X,
y,
scorer=scorers,
train=train,
test=test,
parameters={},
fit_params={},
verbose=self.verbose)[0]
res.append(test_scores)
res_mean = pd.DataFrame(res).mean(axis=0)
try:
# show only last step of pipeline for simplicity
name = nice_repr(estimator.steps[-1][1])
except AttributeError:
name = nice_repr(estimator)
if self.verbose:
print("Running {}".format(name))
print(_format_scores(res_mean))
res_mean.name = name
self.log_.append(res_mean)
return res_mean
def _wrapped_cross_val_score(sklearn_pipeline, features, target,
cv, scoring_function, sample_weight=None,
groups=None, use_dask=False):
"""Fit estimator and compute scores for a given dataset split.
Parameters
----------
sklearn_pipeline : pipeline object implementing 'fit'
The object to use to fit the data.
features : array-like of shape at least 2D
The data to fit.
target : array-like, optional, default: None
The target variable to try to predict in the case of
supervised learning.
cv: int or cross-validation generator
If CV is a number, then it is the number of folds to evaluate each
pipeline over in k-fold cross-validation during the TPOT optimization
process. If it is an object then it is an object to be used as a
cross-validation generator.
scoring_function : callable
A scorer callable object / function with signature
``scorer(estimator, X, y)``.
sample_weight : array-like, optional
List of sample weights to balance (or un-balanace) the dataset target as needed
groups: array-like {n_samples, }, optional
Group labels for the samples used while splitting the dataset into train/test set
use_dask : bool, default False
Whether to use dask
"""
sample_weight_dict = set_sample_weight(sklearn_pipeline.steps, sample_weight)
features, target, groups = indexable(features, target, groups)
cv = check_cv(cv, target, classifier=is_classifier(sklearn_pipeline))
cv_iter = list(cv.split(features, target, groups))
scorer = check_scoring(sklearn_pipeline, scoring=scoring_function)
if use_dask:
try:
import dask_ml.model_selection # noqa
import dask # noqa
from dask.delayed import Delayed
except ImportError:
msg = "'use_dask' requires the optional dask and dask-ml depedencies."
raise ImportError(msg)
dsk, keys, n_splits = dask_ml.model_selection._search.build_graph(
estimator=sklearn_pipeline,
cv=cv,
scorer=scorer,
candidate_params=[{}],
X=features,
y=target,
groups=groups,
fit_params=sample_weight_dict,
refit=False,
error_score=float('-inf'),
)
cv_results = Delayed(keys[0], dsk)
scores = [cv_results['split{}_test_score'.format(i)]
for i in range(n_splits)]
CV_score = dask.delayed(np.array)(scores)[:, 0]
return dask.delayed(np.nanmean)(CV_score)
else:
try:
with warnings.catch_warnings():
warnings.simplefilter('ignore')
scores = [_fit_and_score(estimator=clone(sklearn_pipeline),
X=features,
y=target,
scorer=scorer,
train=train,
test=test,
verbose=0,
parameters=None,
fit_params=sample_weight_dict)
for train, test in cv_iter]
CV_score = np.array(scores)[:, 0]
return np.nanmean(CV_score)
except TimeoutException:
return "Timeout"
except Exception as e:
return -float('inf')
def _wrapped_cross_val_score(sklearn_pipeline,
features,
target,
cv,
scoring_function,
sample_weight=None,
groups=None,
use_dask=False,
random_state=None):
"""Fit estimator and compute scores for a given dataset split.
Parameters
----------
sklearn_pipeline : pipeline object implementing 'fit'
The object to use to fit the data.
features : array-like of shape at least 2D
The data to fit.
target : array-like, optional, default: None
The target variable to try to predict in the case of
supervised learning.
cv: int or cross-validation generator
If CV is a number, then it is the number of folds to evaluate each
pipeline over in k-fold cross-validation during the TPOT optimization
process. If it is an object then it is an object to be used as a
cross-validation generator.
scoring_function : callable
A scorer callable object / function with signature
``scorer(estimator, X, y)``.
sample_weight : array-like, optional
List of sample weights to balance (or un-balanace) the dataset target as needed
groups: array-like {n_samples, }, optional
Group labels for the samples used while splitting the dataset into train/test set
use_dask : bool, default False
Whether to use dask
"""
sample_weight_dict = set_sample_weight(sklearn_pipeline.steps,
sample_weight)
features, target, groups = indexable(features, target, groups)
scorer = check_scoring(sklearn_pipeline, scoring=scoring_function)
if isinstance(cv, float):
try:
def split(l, test_size=0.2):
random.seed(random_state)
n_total = len(l)
offset = int(n_total * test_size)
if n_total == 0 or offset < 1:
return [], l
random.shuffle(l)
sublist_1 = l[:offset]
sublist_2 = l[offset:]
return sublist_1, sublist_2
train_indices, test_indices = split(list(range(len(features))),
test_size=cv)
with warnings.catch_warnings():
warnings.simplefilter('ignore')
score = _fit_and_score(estimator=clone(sklearn_pipeline),
X=features,
y=target,
scorer=scorer,
train=train_indices,
test=test_indices,
verbose=0,
parameters=None,
fit_params=sample_weight_dict)
return score[0]
except TimeoutException:
return "Timeout"
except Exception as e:
return -float('inf')
cv = check_cv(cv, target, classifier=is_classifier(sklearn_pipeline))
cv_iter = list(cv.split(features, target, groups))
if use_dask:
try:
import dask_ml.model_selection # noqa
import dask # noqa
from dask.delayed import Delayed
except Exception as e:
msg = "'use_dask' requires the optional dask and dask-ml depedencies.\n{}".format(
e)
raise ImportError(msg)
dsk, keys, n_splits = dask_ml.model_selection._search.build_graph(
estimator=sklearn_pipeline,
cv=cv,
scorer=scorer,
candidate_params=[{}],
X=features,
y=target,
groups=groups,
fit_params=sample_weight_dict,
refit=False,
error_score=float('-inf'),
)
cv_results = Delayed(keys[0], dsk)
scores = [
cv_results['split{}_test_score'.format(i)] for i in range(n_splits)
]
CV_score = dask.delayed(np.array)(scores)[:, 0]
return dask.delayed(np.nanmean)(CV_score)
else:
try:
with warnings.catch_warnings():
warnings.simplefilter('ignore')
scores = [
_fit_and_score(estimator=clone(sklearn_pipeline),
X=features,
y=target,
scorer=scorer,
train=train,
test=test,
verbose=0,
parameters=None,
fit_params=sample_weight_dict)
for train, test in cv_iter
]
CV_score = np.array(scores)[:, 0]
return np.nanmean(CV_score)
except TimeoutException:
return "Timeout"
except Exception as e:
return -float('inf')
def _wrapped_cross_val_score(sklearn_pipeline,
features,
target,
cv,
scoring_function,
sample_weight=None,
groups=None,
use_dask=False):
"""Fit estimator and compute scores for a given dataset split.
Parameters
----------
sklearn_pipeline : pipeline object implementing 'fit'
The object to use to fit the data.
features : array-like of shape at least 2D
The data to fit.
target : array-like, optional, default: None
The target variable to try to predict in the case of
supervised learning.
cv: cross-validation generator
Object to be used as a cross-validation generator.
scoring_function : callable
A scorer callable object / function with signature
``scorer(estimator, X, y)``.
sample_weight : array-like, optional
List of sample weights to balance (or un-balanace) the dataset target as needed
groups: array-like {n_samples, }, optional
Group labels for the samples used while splitting the dataset into train/test set
use_dask : bool, default False
Whether to use dask
"""
sample_weight_dict = set_sample_weight(sklearn_pipeline.steps,
sample_weight)
features, target, groups = indexable(features, target, groups)
cv_iter = list(cv.split(features, target, groups))
scorer = check_scoring(sklearn_pipeline, scoring=scoring_function)
if use_dask:
try:
import dask_ml.model_selection # noqa
import dask # noqa
from dask.delayed import Delayed
except Exception as e:
msg = "'use_dask' requires the optional dask and dask-ml depedencies.\n{}".format(
e)
raise ImportError(msg)
dsk, keys, n_splits = dask_ml.model_selection._search.build_graph(
estimator=sklearn_pipeline,
cv=cv,
scorer=scorer,
candidate_params=[{}],
X=features,
y=target,
groups=groups,
fit_params=sample_weight_dict,
refit=False,
error_score=float('-inf'),
)
cv_results = Delayed(keys[0], dsk)
scores = [
cv_results['split{}_test_score'.format(i)] for i in range(n_splits)
]
CV_score = dask.delayed(np.array)(scores)[:, 0]
return dask.delayed(np.nanmean)(CV_score)
else:
try:
with warnings.catch_warnings():
warnings.simplefilter('ignore')
scores = [
_fit_and_score(estimator=clone(sklearn_pipeline),
X=features,
y=target,
scorer=scorer,
train=train,
test=test,
verbose=0,
parameters=None,
error_score='raise',
return_estimator=True,
fit_params=sample_weight_dict)
for train, test in cv_iter
]
if isinstance(scores[0], list): #scikit-learn <= 0.23.2
CV_score = np.array(scores)[:, 0]
elif isinstance(scores[0], dict): # scikit-learn >= 0.24
from sklearn.model_selection._validation import _aggregate_score_dicts
CV_score = _aggregate_score_dicts(scores)["test_scores"]
CV_fitted_pipeline = _aggregate_score_dicts(
scores)["estimator"]
else:
raise ValueError(
"Incorrect output format from _fit_and_score!")
fit_and_score_details = dict()
fit_and_score_details["CV_score_mean"] = np.nanmean(CV_score)
fit_and_score_details[
"CV_fitted_best_pipeline"] = CV_fitted_pipeline[0]
return fit_and_score_details
except TimeoutException:
fit_and_score_details = dict()
fit_and_score_details["CV_score_mean"] = "Timeout"
fit_and_score_details["CV_fitted_best_pipeline"] = None
return fit_and_score_details
except Exception as e:
fit_and_score_details = dict()
fit_and_score_details["CV_score_mean"] = -float('inf')
fit_and_score_details["CV_fitted_best_pipeline"] = None
return fit_and_score_details
def fit_and_score(estimator,
X,
y,
scorer,
train,
test,
para,
fit_params=None,
return_train_score=True,
return_n_test_samples=True,
return_times=True,
return_parameters=True,
error_score='raise',
verbose=True):
'''
Fit an estimator to a dataset and score the performance. The following
methods can currently be applied as preprocessing before fitting in
this order:
1. Select features based on type group.
2. Apply feature imputation.
3. Apply feature selection based on variance of feature among patients.
4. Scale features with e.g. z-scoring.
5. Select features based on a fit with a LASSO model.
6. Select features using PCA.
All of the steps are optional.
Parameters
----------
estimator: sklearn estimator, mandatory
Unfitted estimator which will be fit.
X: array, mandatory
Array containingfor each object (rows) the feature values
(1st Column) and the associated feature label (2nd Column).
y: list(?), mandatory
List containing the labels of the objects.
scorer: sklearn scorer, mandatory
Function used as optimization criterion for the hyperparamater optimization.
train: list, mandatory
Indices of the objects to be used as training set.
test: list, mandatory
Indices of the objects to be used as testing set.
para: dictionary, mandatory
Contains the settings used for the above preprocessing functions
and the fitting. TODO: Create a default object and show the
fields.
fit_params:dictionary, default None
Parameters supplied to the estimator for fitting. See the SKlearn
site for the parameters of the estimators.
return_train_score: boolean, default True
Save the training score to the final SearchCV object.
return_n_test_samples: boolean, default True
Save the number of times each sample was used in the test set
to the final SearchCV object.
return_times: boolean, default True
Save the time spend for each fit to the final SearchCV object.
return_parameters: boolean, default True
Return the parameters used in the final fit to the final SearchCV
object.
error_score: numeric or "raise" by default
Value to assign to the score if an error occurs in estimator
fitting. If set to "raise", the error is raised. If a numeric
value is given, FitFailedWarning is raised. This parameter
does not affect the refit step, which will always raise the error.
verbose: boolean, default=True
If True, print intermediate progress to command line. Warnings are
always printed.
Returns
----------
ret
GroupSel
VarSel
SelectModel
feature_labels
scaler
imputer
pca
'''
# We copy the parameter object so we can alter it and keep the original
para_estimator = para.copy()
# X is a tuple: split in two arrays
feature_values = np.asarray([x[0] for x in X])
feature_labels = np.asarray([x[1] for x in X])
# ------------------------------------------------------------------------
# Groupwise feature selection
if 'SelectGroups' in para_estimator:
if verbose:
print("Selecting groups of features.")
del para_estimator['SelectGroups']
# TODO: more elegant way to solve this
feature_groups = [
"histogram_features", "orientation_features", "patient_features",
"semantic_features", "shape_features", "texture_features",
"coliage_features", 'vessel_features', "phase_features",
"log_features"
]
parameters_featsel = dict()
for group in feature_groups:
if group not in para_estimator:
# Default: do use the group
value = True
else:
value = para_estimator[group]
del para_estimator[group]
parameters_featsel[group] = value
GroupSel = SelectGroups(parameters=parameters_featsel)
GroupSel.fit(feature_labels[0])
if verbose:
print("Original Length: " + str(len(feature_values[0])))
feature_values = GroupSel.transform(feature_values)
if verbose:
print("New Length: " + str(len(feature_values[0])))
feature_labels = GroupSel.transform(feature_labels)
else:
GroupSel = None
# Check whether there are any features left
if len(feature_values[0]) == 0:
# TODO: Make a specific PREDICT exception for this warning.
if verbose:
print(
'[WARNING]: No features are selected! Probably all feature groups were set to False. Parameters:'
)
print para
# Return a zero performance dummy
VarSel = None
scaler = None
SelectModel = None
pca = None
StatisticalSel = None
imputer = None
# Delete the non-used fields
para_estimator = delete_nonestimator_parameters(para_estimator)
ret = [0, 0, 0, 0, 0, para_estimator, para]
return ret, GroupSel, VarSel, SelectModel, feature_labels[
0], scaler, imputer, pca, StatisticalSel
# ------------------------------------------------------------------------
# Feature imputation
if 'Imputation' in para_estimator.keys() and ['Imputation'] == 'True':
imp_type = para_estimator['ImputationMethod']
imp_nn = para_estimator['ImputationNeighbours']
imputer = Imputer(missing_values='NaN',
strategy=imp_type,
n_neighbors=imp_nn,
axis=0)
imputer.fit(feature_values)
feature_values = imputer.transform(feature_values)
else:
imputer = None
if 'Imputation' in para_estimator.keys():
del para_estimator['Imputation']
del para_estimator['ImputationMethod']
del para_estimator['ImputationNeighbours']
# ------------------------------------------------------------------------
# FIXME: When only using LBP feature, X is 3 dimensional with 3rd dimension length 1
if len(feature_values.shape) == 3:
feature_values = np.reshape(
feature_values, (feature_values.shape[0], feature_values.shape[1]))
if len(feature_labels.shape) == 3:
feature_labels = np.reshape(
feature_labels, (feature_labels.shape[0], feature_labels.shape[1]))
# Remove any NaN feature values if these are still left after imputation
feature_values = replacenan(feature_values,
verbose=verbose,
feature_labels=feature_labels[0])
# --------------------------------------------------------------------
# Feature selection based on variance
if para_estimator['Featsel_Variance'] == 'True':
if verbose:
print("Selecting features based on variance.")
if verbose:
print("Original Length: " + str(len(feature_values[0])))
try:
feature_values, feature_labels, VarSel =\
selfeat_variance(feature_values, feature_labels)
except ValueError:
if verbose:
print(
'[WARNING]: No features meet the selected Variance threshold! Skipping selection.'
)
VarSel = None
if verbose:
print("New Length: " + str(len(feature_values[0])))
else:
VarSel = None
del para_estimator['Featsel_Variance']
# Check whether there are any features left
if len(feature_values[0]) == 0:
# TODO: Make a specific PREDICT exception for this warning.
if verbose:
print(
'[WARNING]: No features are selected! Probably you selected a feature group that is not in your feature file. Parameters:'
)
print para
para_estimator = delete_nonestimator_parameters(para_estimator)
# Return a zero performance dummy
scaler = None
SelectModel = None
pca = None
StatisticalSel = None
ret = [0, 0, 0, 0, 0, para_estimator, para]
return ret, GroupSel, VarSel, SelectModel, feature_labels[
0], scaler, imputer, pca, StatisticalSel
# --------------------------------------------------------------------
# Feature selection based on a statistical test
if 'StatisticalTestUse' in para_estimator.keys():
if para_estimator['StatisticalTestUse'] == 'True':
metric = para_estimator['StatisticalTestMetric']
threshold = para_estimator['StatisticalTestThreshold']
if verbose:
print(
"Selecting features based on statistical test. Method {}, threshold {}."
).format(metric, str(round(threshold, 2)))
if verbose:
print("Original Length: " + str(len(feature_values[0])))
StatisticalSel = StatisticalTestThreshold(metric=metric,
threshold=threshold)
StatisticalSel.fit(feature_values, y)
feature_values = StatisticalSel.transform(feature_values)
feature_labels = StatisticalSel.transform(feature_labels)
if verbose:
print("New Length: " + str(len(feature_values[0])))
else:
StatisticalSel = None
del para_estimator['StatisticalTestUse']
del para_estimator['StatisticalTestMetric']
del para_estimator['StatisticalTestThreshold']
else:
StatisticalSel = None
# Check whether there are any features left
if len(feature_values[0]) == 0:
# TODO: Make a specific PREDICT exception for this warning.
if verbose:
print(
'[WARNING]: No features are selected! Probably you selected a feature group that is not in your feature file. Parameters:'
)
print para
para_estimator = delete_nonestimator_parameters(para_estimator)
# Return a zero performance dummy
scaler = None
SelectModel = None
pca = None
ret = [0, 0, 0, 0, 0, para_estimator, para]
return ret, GroupSel, VarSel, SelectModel, feature_labels[
0], scaler, imputer, pca, StatisticalSel
# ------------------------------------------------------------------------
# Feature scaling
if 'FeatureScaling' in para_estimator:
if verbose:
print("Fitting scaler and transforming features.")
if para_estimator['FeatureScaling'] == 'z_score':
scaler = StandardScaler().fit(feature_values)
elif para_estimator['FeatureScaling'] == 'minmax':
scaler = MinMaxScaler().fit(feature_values)
else:
scaler = None
if scaler is not None:
feature_values = scaler.transform(feature_values)
del para_estimator['FeatureScaling']
else:
scaler = None
# ------------------------------------------------------------------------
# Perform feature selection using a model
if 'SelectFromModel' in para_estimator.keys(
) and para_estimator['SelectFromModel'] == 'True':
if verbose:
print("Selecting features using lasso model.")
# Use lasso model for feature selection
# First, draw a random value for alpha and the penalty ratio
alpha = scipy.stats.uniform(loc=0.0, scale=1.5).rvs()
# l1_ratio = scipy.stats.uniform(loc=0.5, scale=0.4).rvs()
# Create and fit lasso model
lassomodel = Lasso(alpha=alpha)
lassomodel.fit(feature_values, y)
# Use fit to select optimal features
SelectModel = SelectFromModel(lassomodel, prefit=True)
if verbose:
print("Original Length: " + str(len(feature_values[0])))
feature_values = SelectModel.transform(feature_values)
if verbose:
print("New Length: " + str(len(feature_values[0])))
feature_labels = SelectModel.transform(feature_labels)
else:
SelectModel = None
if 'SelectFromModel' in para_estimator.keys():
del para_estimator['SelectFromModel']
# ----------------------------------------------------------------
# PCA dimensionality reduction
# Principle Component Analysis
if 'UsePCA' in para_estimator.keys() and ['UsePCA'] == 'True':
print('Fitting PCA')
if para_estimator['PCAType'] == '95variance':
# Select first X components that describe 95 percent of the explained variance
pca = PCA(n_components=None)
pca.fit(feature_values)
evariance = pca.explained_variance_ratio
num = 0
sum = 0
while sum < 0.95:
sum += evariance[num]
num += 1
# Make a PCA based on the determined amound of components
pca = PCA(n_components=num)
pca.fit(feature_values)
feature_values = pca.transform(feature_values)
feature_labels = pca.transform(feature_labels)
else:
# Assume a fixed number of components
n_components = int(para_estimator['PCAType'])
pca = PCA(n_components=n_components)
pca.fit(feature_values)
feature_values = pca.transform(feature_values)
feature_labels = pca.transform(feature_labels)
else:
pca = None
if 'UsePCA' in para_estimator.keys():
del para_estimator['UsePCA']
del para_estimator['PCAType']
# ----------------------------------------------------------------
# Fitting and scoring
# Only when using fastr this is an entry
if 'Number' in para_estimator.keys():
del para_estimator['Number']
# For certainty, we delete all parameters again
para_estimator = delete_nonestimator_parameters(para_estimator)
ret = _fit_and_score(estimator, feature_values, y, scorer, train, test,
verbose, para_estimator, fit_params,
return_train_score, return_parameters,
return_n_test_samples, return_times, error_score)
# Paste original parameters in performance
ret.append(para)
return ret, GroupSel, VarSel, SelectModel, feature_labels[
0], scaler, imputer, pca, StatisticalSel
def _fit_and_score_ckpt(workdir=None,
checkpoint=True,
force_refresh=False,
**fit_and_score_kwargs):
"""Fit estimator and compute scores for a given dataset split.
This function wraps
:func:`sklearn:sklearn.model_selection._validation._fit_and_score`,
while also saving checkpoint files containing the estimator, paramters,
This is useful if fitting and scoring is costly or if it is being
performed within a large cross-validation experiment.
In avoid collisions with scores computed for other CV splits, this
function computes a hash from a nested dictionary containing all keyword
arguments as well as estimator parameters. It then saves the scores and
parameters in <hash>_params.h5 and the estimator itself in
<hash>_estimator.pkl
Parameters
----------
workdir : path-like object, default=None
A string or :term:`python:path-like-object` indicating the directory
in which to store checkpoint files
checkpoint : bool, default=True
If True, checkpoint the parameters, estimators, and scores.
force_refresh : bool, default=False
If True, recompute scores even if the checkpoint file already exists.
Otherwise, load scores from checkpoint files and return.
**fit_and_score_kwargs : kwargs
Key-word arguments passed to
:func:`sklearn:sklearn.model_selection._validation._fit_and_score`
Returns
-------
train_scores : dict of scorer name -> float
Score on training set (for all the scorers),
returned only if `return_train_score` is `True`.
test_scores : dict of scorer name -> float
Score on testing set (for all the scorers).
n_test_samples : int
Number of test samples.
fit_time : float
Time spent for fitting in seconds.
score_time : float
Time spent for scoring in seconds.
parameters : dict or None
The parameters that have been evaluated.
estimator : estimator object
The fitted estimator
"""
if not checkpoint:
return _fit_and_score(**fit_and_score_kwargs)
if workdir is None:
raise ValueError(
"If checkpoint is True, you must supply a working directory "
"through the ``workdir`` argument.")
estimator = fit_and_score_kwargs.pop("estimator", None)
estimator_params = _serialize_estimator_params(estimator.get_params())
all_params = {
"estimator_params": estimator_params,
"fit_and_score_kwargs": fit_and_score_kwargs,
}
cv_hash = hashlib.md5(
json.dumps(all_params, sort_keys=True, ensure_ascii=True,
default=str).encode()).hexdigest()
h5_file = os.path.join(workdir, cv_hash + "_params.h5")
pkl_file = os.path.join(workdir, cv_hash + "_estimator.pkl")
if not force_refresh and os.path.exists(h5_file):
ckpt_dict = ddio.load(h5_file)
scores = ckpt_dict["scores"]
if fit_and_score_kwargs.get("return_estimator", False):
with open(pkl_file, "rb") as fp:
estimator = pickle.load(fp)
scores.append(estimator)
return scores
else:
scores = _fit_and_score(estimator, **fit_and_score_kwargs)
os.makedirs(workdir, exist_ok=True)
if fit_and_score_kwargs.get("return_estimator", False):
estimator = scores[-1]
with open(pkl_file, "wb") as fp:
pickle.dump(estimator, fp)
ckpt_scores = scores[:-1]
if isinstance(estimator, Pipeline):
model = estimator.steps[-1]
else:
model = estimator
estimator_params = _serialize_estimator_params(
estimator.get_params())
fitted_params = {
"alpha_": getattr(model, "alpha_", None),
"alphas_": getattr(model, "alphas_", None),
"l1_ratio_": getattr(model, "l1_ratio_", None),
"mse_path_": getattr(model, "mse_path_", None),
"scoring_path_": getattr(model, "scoring_path_", None),
"intercept_": getattr(model, "intercept_", None),
"coef_": getattr(model, "coef_", None),
}
else:
estimator_params = None
fitted_params = None
ckpt_scores = scores
fit_and_score_kwargs.pop("X")
fit_and_score_kwargs.pop("y")
if "scorer" in fit_and_score_kwargs:
fit_and_score_kwargs["scorer"] = list(
fit_and_score_kwargs["scorer"].keys())
ckpt_dict = {
"scores": ckpt_scores,
"fit_and_score_kwargs": fit_and_score_kwargs,
"estimator_params": estimator_params,
"fitted_params": fitted_params,
}
ddio.save(h5_file, ckpt_dict)
return scores
def fit_and_score(estimator,
X,
y,
scorer,
train,
test,
para,
fit_params=None,
return_train_score=True,
return_n_test_samples=True,
return_times=True,
return_parameters=True,
error_score='raise',
verbose=True):
'''
Fit an estimator to a dataset and score the performance. The following
methods can currently be applied as preprocessing before fitting in
this order:
1. Apply feature selection based on type group.
2. Apply feature selection based on variance of feature among patients.
3. Scale features with e.g. z-scoring.
Parameters
----------
estimator: sklearn estimator, mandatory
Unfitted estimator which will be fit.
X: array, mandatory
Array containing the feature values (columns) for each object (rows).
y: list(?), mandatory
List containing the labels of the objects.
scorer: sklearn scorer, mandatory
Function used as optimization criterion for the hyperparamater optimization.
train: list, mandatory
Indices of the objects to be used as training set.
test: list, mandatory
Indices of the objects to be used as testing set.
para: dictionary, mandatory
Contains the settings used for the above preprocessing functions
and the fitting. TODO: Create a default object and show the
fields.
fit_params:dictionary, default None
Parameters supplied to the estimator for fitting. See the SKlearn
site for the parameters of the estimators.
return_train_score: boolean, default True
Save the training score to the final SearchCV object.
return_n_test_samples: boolean, default True
Save the number of times each sample was used in the test set
to the final SearchCV object.
return_times: boolean, default True
Save the time spend for each fit to the final SearchCV object.
return_parameters: boolean, default True
Return the parameters used in the final fit to the final SearchCV
object.
error_score: numeric or "raise" by default
Value to assign to the score if an error occurs in estimator
fitting. If set to "raise", the error is raised. If a numeric
value is given, FitFailedWarning is raised. This parameter
does not affect the refit step, which will always raise the error.
verbose: boolean, default=True
If True, print intermediate progress to command line. Warnings are
always printed.
'''
pca = None
# We copy the parameter object so we can alter it and keep the original
para_estimator = para.copy()
for i in range(len(X)):
if len(X[i][0]) != len(X[i][0]):
raise IOError(
'Length of features values ({}) does not match length of feature labels ({})\n \
Check CalcFeatures'.format(len(X[i][0]),
len(X[0][1])))
# X is a tuple: split in two arrays
feature_values = np.asarray([x[0] for x in X])
feature_labels = np.asarray([x[1] for x in X])
# Perform feature selection if required
if 'SelectGroups' in para_estimator:
if verbose:
print("Selecting groups of features.")
del para_estimator['SelectGroups']
# TODO: more elegant way to solve this
feature_groups = [
"histogram_features", "orientation_features", "patient_features",
"semantic_features", "shape_features", "texture_features",
"coliage_features", 'vessel_features', "phase_features",
"log_features"
]
parameters_featsel = dict()
for group in feature_groups:
if group not in para_estimator:
# Default: do use the group
value = True
else:
value = para_estimator[group]
del para_estimator[group]
parameters_featsel[group] = value
GroupSel = SelectGroups(parameters=parameters_featsel)
GroupSel.fit(feature_labels[0])
if verbose:
print("Original Length: " + str(len(feature_values[0])))
feature_values = GroupSel.transform(feature_values)
if verbose:
print("New Length: " + str(len(feature_values[0])))
feature_labels = GroupSel.transform(feature_labels)
else:
GroupSel = None
if len(feature_values[0]) == 0:
# TODO: Make a specific PREDICT exception for this warning.
print(
'[WARNING]: No features are selected! Probably all feature groups were set to False. Parameters:'
)
print para
# Return a zero performance dummy
VarSel = None
scaler = None
SelectModel = None
# Delete the non-used fields
if 'Featsel_Variance' in para_estimator.keys():
del para_estimator['Featsel_Variance']
if 'FeatureScaling' in para_estimator.keys():
del para_estimator['FeatureScaling']
ret = [0, 0, 0, 0, 0, para_estimator, para]
else:
# FIXME: When only using LBP feature, X is 3 dimensional with 3rd dimension length 1
if len(feature_values.shape) == 3:
feature_values = np.reshape(
feature_values,
(feature_values.shape[0], feature_values.shape[1]))
if len(feature_labels.shape) == 3:
feature_labels = np.reshape(
feature_labels,
(feature_labels.shape[0], feature_labels.shape[1]))
if para_estimator['Featsel_Variance'] == 'True':
if verbose:
print("Selecting features based on variance.")
if verbose:
print("Original Length: " + str(len(feature_values[0])))
try:
feature_values, feature_labels, VarSel =\
selfeat_variance(feature_values, feature_labels)
except ValueError:
print(
'[WARNING]: No features meet the selected Variance threshold! Skipping selection.'
)
VarSel = None
if verbose:
print("New Length: " + str(len(feature_values[0])))
else:
VarSel = None
del para_estimator['Featsel_Variance']
# Fit and score the classifier
if len(feature_values[0]) == 0:
# TODO: Make a specific PREDICT exception for this warning.
print(
'[WARNING]: No features are selected! Probably you selected a feature group that is not in your feature file. Parameters:'
)
print para
# Return a zero performance dummy
scaler = None
SelectModel = None
ret = [0, 0, 0, 0, 0, para_estimator, para]
del para_estimator['FeatureScaling']
else:
if 'FeatureScaling' in para_estimator:
if verbose:
print("Fitting scaler and transforming features.")
if para_estimator['FeatureScaling'] == 'z_score':
scaler = StandardScaler().fit(feature_values)
elif para_estimator['FeatureScaling'] == 'minmax':
scaler = MinMaxScaler().fit(feature_values)
else:
scaler = None
if scaler is not None:
feature_values = scaler.transform(feature_values)
else:
scaler = None
del para_estimator['FeatureScaling']
# Only when using fastr this is an entry
if 'Number' in para_estimator.keys():
del para_estimator['Number']
# Perform feature selection using a model
para_estimator['SelectFromModel'] = False
if para_estimator['SelectFromModel']:
if verbose:
print("Selecting features using lasso model.")
# Use lasso model for feature selection
# First, draw a random value for alpha and the penalty ratio
alpha = scipy.stats.uniform(loc=0.5, scale=1.5).rvs()
# l1_ratio = scipy.stats.uniform(loc=0.5, scale=0.4).rvs()
# Create and fit lasso model
lassomodel = Lasso(alpha=alpha)
lassomodel.fit(feature_values, y)
# Use fit to select optimal features
SelectModel = SelectFromModel(lassomodel, prefit=True)
if verbose:
print("Original Length: " + str(len(feature_values[0])))
feature_values = SelectModel.transform(feature_values)
if verbose:
print("New Length: " + str(len(feature_values[0])))
feature_labels = SelectModel.transform(feature_labels)
else:
SelectModel = None
del para_estimator['SelectFromModel']
#Principle Component Analysis
do_PCA = False
n_components = 39
if len(feature_values[0]) < n_components:
n_components = len(feature_values[0])
if do_PCA:
pca = PCA(n_components=n_components)
pca.fit(feature_values)
feature_values = pca.transform(feature_values)
print('Fitting PCA')
ret = _fit_and_score(estimator, feature_values, y, scorer, train,
test, verbose, para_estimator, fit_params,
return_train_score, return_parameters,
return_n_test_samples, return_times,
error_score)
# Paste original parameters in performance
ret.append(para)
return ret, GroupSel, VarSel, SelectModel, feature_labels[0], scaler
def grid_search_cv(
clf,
train_indices,
n_folds,
param_grid,
kernel_matrices,
):
"""
Internal grid search routine for a set of kernel matrices. The
routine will use a pre-defined set of train indices to use for
the grid search. Other indices will *not* be considered. Thus,
information leakage is prevented.
:param clf: Classifier to fit
:param train_indices: Indices permitted to be used for cross-validation
:param n_folds: Number of folds for the cross-validation
:param param_grid: Parameters for the grid search
:param kernel_matrices: Kernel matrices to check; each one of them
is assumed to represent a different choice of parameter. They will
*all* be checked iteratively by the routine.
:return: Best classifier, i.e. the classifier with the best
parameters. Needs to be refit prior to predicting labels on
the test data set. Moreover, the best-performing matrix, in
terms of the grid search, is returned. It has to be used in
all subsequent prediction tasks. Additionally, the function
also returns a dictionary of the best parameters.
"""
y = kernel_matrices["y"][train_indices]
best_clf = None
best_accuracy = 0.0
best_parameters = {}
# iterate over parameters in most outer loop to avoid issues
# with matrix normalization (when we loop over the matrices)
# in the next loop, then parameters['normalize'] can only
# be either True or False.
for parameters in list(param_grid):
for K_param, K in kernel_matrices.items():
# Skip labels; we could also remove them from the set of
# matrices but this would make the function inconsistent
# because it should *not* fiddle with the input data set
# if it can be avoided.
if K_param == "y":
continue
# This ensures that we *cannot* access the test indices,
# even if we try :)
K = K[train_indices, :][:, train_indices]
# normalize kernel matrix if parameters['normalize'] == True
if parameters["normalize"]:
K = normalize(K)
# Remove the parameter because it does not pertain to
# the classifier below.
clf_parameters = {
key: value
for key, value in parameters.items()
if key not in ["normalize"]
}
# we have to create a new cv instance for each parameter
# tuple, because StratifiedKFold returns a generator
cv = StratifiedKFold(
n_splits=n_folds,
shuffle=True,
random_state=42, # TODO: make configurable
)
# initialize empty list to store fold accuracies of the
# current parameters in.
accuracy_list = []
# From this point on, `train_index` and `test_index` are supposed to
# be understood *relative* to the input training indices.
for train_index, test_index in cv.split(train_indices, y):
accuracy, params = _fit_and_score(
clone(clf),
K,
y,
scorer=make_scorer(accuracy_score),
train=train_index,
test=test_index,
verbose=0,
parameters=clf_parameters,
fit_params=None, # No additional parameters for `fit()`
return_parameters=True,
)
accuracy_list.append(accuracy)
# compute accuracy mean of current parameters to compare to
# previously best result.
accuracy_mean = np.mean(accuracy_list)
# Note that when storing the best parameters, we can re-use
# the original grid because we want to know about this
# normalization.
if accuracy_mean > best_accuracy:
best_clf = clone(clf).set_params(**params)
best_accuracy = accuracy_mean
# Make a copy of the dictionary to ensure that we are
# not updating it with parameters that cannot be used
# in the grid search (such as `K`).
best_parameters = dict(parameters)
# Update kernel matrix parameter to indicate which
# matrix was used to obtain these results. The key
# will also be returned later on.
best_parameters["K"] = K_param
# Retrieve the kernel matrix of the best performing
# model and normalize if `best_parameters['normalize']`
# is True
best_K = kernel_matrices[best_parameters["K"]]
if best_parameters["normalize"]:
best_K = normalize(best_K)
return best_clf, best_K, best_parameters
