predictions) # Calculating the Recall score.
recall_list.append(recall)
kappa = cohen_kappa_score(activity_shuffled,
predictions) # Calculating the Kappa score.
kappa_list.append(kappa)
#@ GINI Features Importance:
gini_importances[
"Run" +
str(seed)] = model.feature_importances_ # Features Importance: GINI.
#@ PERMUTATION Features Importance:
permutation_important = permutation_importance(
model,
data,
activity_shuffled,
n_repeats=5, # Permutation Importance.
n_jobs=-1,
random_state=seed)
permutation_importances["Run" + str(
seed
)] = permutation_important.importances_mean # Features Importance: PERMUTATION.
# 1. Evaluation Metrics:
# I will calculate the Evaluation metrics such as OOB score, Recall score and Kappa score using the whole dataset.
# I will be changing the seed of Random Forest Classifier along with the loop. The values of seed starts from 123
# and goes up to 148 which will also be the range of Iteration of the loop which is 25 Iterations in total.
#@ Dictionary of Evaluation Metrics:
scores = {
"OOB Score": oob_list,
X, y = data.data, data.target
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
clf = RandomForestClassifier(n_estimators=100, random_state=42)
clf.fit(X_train, y_train)
print("Accuracy on test data: {:.2f}".format(clf.score(X_test, y_test)))
# %%
# Next, we plot the tree based feature importance and the permutation
# importance. The permutation importance plot shows that permuting a feature
# drops the accuracy by at most `0.012`, which would suggest that none of the
# features are important. This is in contradiction with the high test accuracy
# computed above: some feature must be important. The permutation importance
# is calculated on the training set to show how much the model relies on each
# feature during training.
result = permutation_importance(clf, X_train, y_train, n_repeats=10, random_state=42)
perm_sorted_idx = result.importances_mean.argsort()
tree_importance_sorted_idx = np.argsort(clf.feature_importances_)
tree_indices = np.arange(0, len(clf.feature_importances_)) + 0.5
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 8))
ax1.barh(tree_indices, clf.feature_importances_[tree_importance_sorted_idx], height=0.7)
ax1.set_yticks(tree_indices)
ax1.set_yticklabels(data.feature_names[tree_importance_sorted_idx])
ax1.set_ylim((0, len(clf.feature_importances_)))
ax2.boxplot(
result.importances[perm_sorted_idx].T,
vert=False,
labels=data.feature_names[perm_sorted_idx],
)
print('Target on test data', predict_test)
# Accuracy Score on test dataset
accuracy_test = accuracy_score(y_Test, predict_test)
print('accuracy_score on test dataset : ', accuracy_test)
print("R2")
print(r2_score(y_Test, predict_test))
print(classification_report(y_Test, predict_test))
def plot(classifier, x, y, title):
class_names = ["Not Defaulted", "Defaulted"]
disp = plot_confusion_matrix(classifier,
x,
y,
display_labels=class_names,
cmap=plt.cm.PuRd,
values_format='')
disp.ax_.set_title(title)
plt.show()
plot(model, x_Test, y_Test, "Naive Bayes Model-scaled")
imps = permutation_importance(model, x_Test, y_Test)
print(imps.importances_mean)
# In[ ]:
def main():
# Lazy import libraries
from rlearnlib.utils import (
predefined_estimators,
load_training_data,
save_training_data,
option_to_list,
scoring_metrics,
check_class_weights,
)
from rlearnlib.raster import RasterStack
try:
import sklearn
if sklearn.__version__ < "0.20":
gs.fatal(
"Package python3-scikit-learn 0.20 or newer is not installed")
except ImportError:
gs.fatal("Package python3-scikit-learn 0.20 or newer is not installed")
try:
import pandas as pd
except ImportError:
gs.fatal("Package python3-pandas 0.25 or newer is not installed")
# parser options ----------------------------------------------------------
group = options["group"]
training_map = options["training_map"]
training_points = options["training_points"]
field = options["field"]
model_save = options["save_model"]
model_name = options["model_name"]
hyperparams = {
"penalty": options["penalty"],
"alpha": options["alpha"],
"l1_ratio": options["l1_ratio"],
"C": options["c"],
"epsilon": options["epsilon"],
"min_samples_leaf": options["min_samples_leaf"],
"n_estimators": options["n_estimators"],
"learning_rate": options["learning_rate"],
"subsample": options["subsample"],
"max_depth": options["max_depth"],
"max_features": options["max_features"],
"n_neighbors": options["n_neighbors"],
"weights": options["weights"],
"hidden_layer_sizes": options["hidden_units"],
}
cv = int(options["cv"])
group_raster = options["group_raster"]
importances = flags["f"]
preds_file = options["preds_file"]
classif_file = options["classif_file"]
fimp_file = options["fimp_file"]
param_file = options["param_file"]
norm_data = flags["s"]
random_state = int(options["random_state"])
load_training = options["load_training"]
save_training = options["save_training"]
n_jobs = int(options["n_jobs"])
balance = flags["b"]
category_maps = option_to_list(options["category_maps"])
# define estimator --------------------------------------------------------
hyperparams, param_grid = process_param_grid(hyperparams)
estimator, mode = predefined_estimators(model_name, random_state, n_jobs,
hyperparams)
# remove dict keys that are incompatible for the selected estimator
estimator_params = estimator.get_params()
param_grid = {
key: value
for key, value in param_grid.items() if key in estimator_params
}
scoring, search_scorer = scoring_metrics(mode)
# checks of input options -------------------------------------------------
if (mode == "classification" and balance is True
and model_name not in check_class_weights()):
gs.warning(model_name + " does not support class weights")
balance = False
if mode == "regression" and balance is True:
gs.warning(
"Balancing of class weights is only possible for classification")
balance = False
if classif_file:
if cv <= 1:
gs.fatal("Output of cross-validation global accuracy requires "
"cross-validation cv > 1")
if not os.path.exists(os.path.dirname(classif_file)):
gs.fatal("Directory for output file {} does not exist".format(
classif_file))
# feature importance file selected but no cross-validation scheme used
if importances:
if sklearn.__version__ < "0.22":
gs.fatal("Feature importances calculation requires scikit-learn "
"version >= 0.22")
if fimp_file:
if importances is False:
gs.fatal(
'Output of feature importance requires the "f" flag to be set')
if not os.path.exists(os.path.dirname(fimp_file)):
gs.fatal("Directory for output file {} does not exist".format(
fimp_file))
# predictions file selected but no cross-validation scheme used
if preds_file:
if cv <= 1:
gs.fatal("Output of cross-validation predictions requires "
"cross-validation cv > 1")
if not os.path.exists(os.path.dirname(preds_file)):
gs.fatal("Directory for output file {} does not exist".format(
preds_file))
# define RasterStack ------------------------------------------------------
stack = RasterStack(group=group)
if category_maps is not None:
stack.categorical = category_maps
# extract training data ---------------------------------------------------
if load_training != "":
X, y, cat, class_labels, group_id = load_training_data(load_training)
if class_labels is not None:
a = pd.DataFrame({"response": y, "labels": class_labels})
a = a.drop_duplicates().values
class_labels = {k: v for (k, v) in a}
else:
gs.message("Extracting training data")
if group_raster != "":
stack.append(group_raster)
if training_map != "":
X, y, cat = stack.extract_pixels(training_map)
y = y.flatten()
with RasterRow(training_map) as src:
if mode == "classification":
src_cats = {v: k for (k, v, m) in src.cats}
class_labels = {k: k for k in np.unique(y)}
class_labels.update(src_cats)
else:
class_labels = None
elif training_points != "":
X, y, cat = stack.extract_points(training_points, field)
y = y.flatten()
if y.dtype in (np.object_, np.object):
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
y = le.fit_transform(y)
class_labels = {k: v for (k, v) in enumerate(le.classes_)}
else:
class_labels = None
# take group id from last column and remove from predictors
if group_raster != "":
group_id = X[:, -1]
X = np.delete(X, -1, axis=1)
stack.drop(group_raster)
else:
group_id = None
# check for labelled pixels and training data
if y.shape[0] == 0 or X.shape[0] == 0:
gs.fatal("No training pixels or pixels in imagery group ...check "
"computational region")
from sklearn.utils import shuffle
if group_id is None:
X, y, cat = shuffle(X, y, cat, random_state=random_state)
else:
X, y, cat, group_id = shuffle(X,
y,
cat,
group_id,
random_state=random_state)
if save_training != "":
save_training_data(save_training, X, y, cat, class_labels,
group_id, stack.names)
# cross validation settings -----------------------------------------------
# inner resampling method (cv=2)
from sklearn.model_selection import GridSearchCV, StratifiedKFold, GroupKFold, KFold
if any(param_grid) is True:
if group_id is None and mode == "classification":
inner = StratifiedKFold(n_splits=3)
elif group_id is None and mode == "regression":
inner = KFold(n_splits=3)
else:
inner = GroupKFold(n_splits=3)
else:
inner = None
# outer resampling method (cv=cv)
if cv > 1:
if group_id is None and mode == "classification":
outer = StratifiedKFold(n_splits=cv)
elif group_id is None and mode == "regression":
outer = KFold(n_splits=cv)
else:
outer = GroupKFold(n_splits=cv)
# modify estimators that take sample_weights ------------------------------
if balance is True:
from sklearn.utils import compute_class_weight
class_weights = compute_class_weight(class_weight="balanced",
classes=(y),
y=y)
fit_params = {"sample_weight": class_weights}
else:
class_weights = None
fit_params = {}
# preprocessing -----------------------------------------------------------
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
# standardization
if norm_data is True and category_maps is None:
scaler = StandardScaler()
trans = ColumnTransformer(
remainder="passthrough",
transformers=[("scaling", scaler, np.arange(0, stack.count))],
)
# one-hot encoding
elif norm_data is False and category_maps is not None:
enc = OneHotEncoder(handle_unknown="ignore", sparse=False)
trans = ColumnTransformer(remainder="passthrough",
transformers=[("onehot", enc,
stack.categorical)])
# standardization and one-hot encoding
elif norm_data is True and category_maps is not None:
scaler = StandardScaler()
enc = OneHotEncoder(handle_unknown="ignore", sparse=False)
trans = ColumnTransformer(
remainder="passthrough",
transformers=[
("onehot", enc, stack.categorical),
(
"scaling",
scaler,
np.setxor1d(range(stack.count),
stack.categorical).astype("int"),
),
],
)
# combine transformers
if norm_data is True or category_maps is not None:
estimator = Pipeline([("preprocessing", trans),
("estimator", estimator)])
param_grid = wrap_named_step(param_grid)
fit_params = wrap_named_step(fit_params)
if any(param_grid) is True:
estimator = GridSearchCV(
estimator=estimator,
param_grid=param_grid,
scoring=search_scorer,
n_jobs=n_jobs,
cv=inner,
)
# estimator training ------------------------------------------------------
gs.message(os.linesep)
gs.message(("Fitting model using " + model_name))
if balance is True and group_id is not None:
estimator.fit(X, y, groups=group_id, **fit_params)
elif balance is True and group_id is None:
estimator.fit(X, y, **fit_params)
else:
estimator.fit(X, y)
# message best hyperparameter setup and optionally save using pandas
if any(param_grid) is True:
gs.message(os.linesep)
gs.message("Best parameters:")
optimal_pars = [
(k.replace("estimator__", "").replace("selection__", "") + " = " +
str(v)) for (k, v) in estimator.best_params_.items()
]
for i in optimal_pars:
gs.message(i)
if param_file != "":
param_df = pd.DataFrame(estimator.cv_results_)
param_df.to_csv(param_file)
# cross-validation --------------------------------------------------------
if cv > 1:
from sklearn.metrics import classification_report
from sklearn import metrics
if (mode == "classification"
and cv > np.histogram(y, bins=np.unique(y))[0].min()):
gs.message(os.linesep)
gs.fatal("Number of cv folds is greater than number of samples in "
"some classes ")
gs.message(os.linesep)
gs.message("Cross validation global performance measures......:")
if (mode == "classification" and len(np.unique(y)) == 2
and all([0, 1] == np.unique(y))):
scoring["roc_auc"] = metrics.roc_auc_score
from sklearn.model_selection import cross_val_predict
preds = cross_val_predict(
estimator=estimator,
X=X,
y=y,
groups=group_id,
cv=outer,
n_jobs=n_jobs,
fit_params=fit_params,
)
test_idx = [test for train, test in outer.split(X, y)]
n_fold = np.zeros((0, ))
for fold in range(outer.get_n_splits()):
n_fold = np.hstack((n_fold, np.repeat(fold,
test_idx[fold].shape[0])))
preds = {"y_pred": preds, "y_true": y, "cat": cat, "fold": n_fold}
preds = pd.DataFrame(data=preds,
columns=["y_pred", "y_true", "cat", "fold"])
gs.message(os.linesep)
gs.message("Global cross validation scores...")
gs.message(os.linesep)
gs.message("Metric \t Mean \t Error")
for name, func in scoring.items():
score_mean = (preds.groupby("fold").apply(
lambda x: func(x["y_true"], x["y_pred"])).mean())
score_std = (preds.groupby("fold").apply(
lambda x: func(x["y_true"], x["y_pred"])).std())
gs.message(name + "\t" + str(score_mean.round(3)) + "\t" +
str(score_std.round(3)))
if mode == "classification":
gs.message(os.linesep)
gs.message("Cross validation class performance measures......:")
report_str = classification_report(
y_true=preds["y_true"],
y_pred=preds["y_pred"],
sample_weight=class_weights,
output_dict=False,
)
report = classification_report(
y_true=preds["y_true"],
y_pred=preds["y_pred"],
sample_weight=class_weights,
output_dict=True,
)
report = pd.DataFrame(report)
gs.message(report_str)
if classif_file != "":
report.to_csv(classif_file, mode="w", index=True)
# write cross-validation predictions to csv file
if preds_file != "":
preds.to_csv(preds_file, mode="w", index=False)
text_file = open(preds_file + "t", "w")
text_file.write('"Real", "Real", "integer", "integer"')
text_file.close()
# feature importances -----------------------------------------------------
if importances is True:
from sklearn.inspection import permutation_importance
fimp = permutation_importance(
estimator,
X,
y,
scoring=search_scorer,
n_repeats=5,
n_jobs=n_jobs,
random_state=random_state,
)
feature_names = deepcopy(stack.names)
feature_names = [i.split("@")[0] for i in feature_names]
fimp = pd.DataFrame({
"feature": feature_names,
"importance": fimp["importances_mean"],
"std": fimp["importances_std"],
})
gs.message(os.linesep)
gs.message("Feature importances")
gs.message("Feature" + "\t" + "Score")
for index, row in fimp.iterrows():
gs.message(row["feature"] + "\t" + str(row["importance"]) + "\t" +
str(row["std"]))
if fimp_file != "":
fimp.to_csv(fimp_file, index=False)
# save the fitted model
import joblib
joblib.dump((estimator, y, class_labels), model_save)
features = np.array(('pe', 'closestPMT', 'n100', 'reco_wall_r', 'reco_wall_z'))
print(features)
feature_importance = gbr.feature_importances_
print(feature_importance)
sorted_idx = np.argsort(feature_importance)
pos = np.arange(sorted_idx.shape[0]) + .5
fig = plt.figure(figsize=(12, 6))
plt.subplot(1, 2, 1)
plt.barh(pos, feature_importance[sorted_idx], align='center')
plt.yticks(pos, features[sorted_idx])
plt.title('Feature Importance (MDI)')
print(features[sorted_idx])
result = permutation_importance(gbr, reco_test, mc_test, n_repeats=10,
random_state=42, n_jobs=2)
print(result)
sorted_idx = result.importances_mean.argsort()
plt.subplot(1, 2, 2)
plt.boxplot(result.importances[sorted_idx].T,
vert=False, labels=features[sorted_idx])
plt.title("Permutation Importance (test set)")
fig.tight_layout()
plt.show()
"""
infile = 'wbls_220000_final_ml_output.csv'
file = open(str(infile))
file_read = pd.read_csv(file)
df = file_read[['TrueEnergy', 'RecoE', 'pe']]
df1=df.iloc[:, ga.best_fen]
cros=model(X_train=df1, y_train=out, method=method)
print("кросс-валидация=", cros)
print("Количество признаков ", np.sum(ga.best_fen))
X_train, X_test, y_train, y_test = train_test_split(
df1, out, test_size=0.33, random_state=42, stratify=out)
out_model=model(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test,
method=method)
y_pred_test=out_model["y_mod_test"]
y_pred_train=out_model["y_mod_train"]
#оценка значимости признаков
results = permutation_importance(out_model["model"], X_train, y_train,
scoring='f1_macro', n_repeats=10)
df_fi1=pd.DataFrame( results.importances_mean, columns=["important"])
df_fi1["features"]=list(X_train.columns)
df_fi1["method"]=method
df_fi1=df_fi1[["method", "features", "important"]]
df_fi.append(df_fi1)
#print("точность по тестовой выборке=", out_model["f1_test"])
#print("точность по обучающей выборке=", out_model["f1_train"])
#plt.figure()
#plt.plot(y_pred_test, "r*", label="Модель")
#plt.plot(y_test.values, "b*", label="Выборка")
#plt.title("Test")
def calculate_permutation_importance(train="b234", test="b261", method="svmk"):
"""
Calculate the permutation importance of each variable to predict RR-Lyrae stars.
Persist the results in the local filesystem as pkl files.
Parameters
----------
train: id of the tile to be used as training dataset
test: id of the tile to be used as test dataset
method: either "svmk", "rf" or "linear"
References
----------
[1] https://scikit-learn.org/stable/modules/permutation_importance.html
"""
X, y = CARPYNCHO.retrieve_tile(train, "full")
Xt, yt = CARPYNCHO.retrieve_tile(test, "full")
if method == "rf":
clf = RandomForestClassifier(n_estimators=400,
criterion="entropy",
min_samples_leaf=2,
max_features="sqrt",
n_jobs=7)
clf.fit(X, y)
if method == "linear":
clf = Pipeline([('disc',
KBinsDiscretizer(
n_bins=get_optimal_parameters_p("svml")["n_bins"],
encode='ordinal',
strategy='quantile')),
('scaler', StandardScaler()),
('clf',
LinearSVC(verbose=3,
max_iter=100000,
C=get_optimal_parameters_p("svml")["C"],
dual=False))])
clf.fit(X, y)
#SVM-K
if method == "svmk":
clf = Pipeline([('disc',
KBinsDiscretizer(
n_bins=get_optimal_parameters_p("svmk")["n_bins"],
encode='ordinal',
strategy='quantile')),
("scaler", StandardScaler()),
("feature_map",
Nystroem(
n_components=300,
gamma=get_optimal_parameters_p("svmk")["gamma"],
)),
("svm",
LinearSVC(
dual=False,
max_iter=100000,
C=get_optimal_parameters_p("svmk")["C"],
))])
clf.fit(X, y)
result = permutation_importance(clf,
Xt,
yt,
scoring="average_precision",
n_repeats=1)
with open(
EXPERIMENTS_OUTPUT_FOLDER_INSPECTION + "Permutation_importance/" +
method + "_RESULT_train=" + train + "test=" + test + ".pkl",
'wb') as output:
pickle.dump(result, output, pickle.HIGHEST_PROTOCOL)
[X_train, X_test, y_train,
y_test] = train_test_split(X,
y,
test_size=.2,
random_state=randint(0, 1000))
model: KNeighborsRegressor = KNeighborsRegressor(n_neighbors=20,
n_jobs=1).fit(
X_train, y_train)
score_test[j] = model.score(X_test, y_test)
score_train[j] = model.score(X_train, y_train)
importances = permutation_importance(model,
X_train,
y_train,
random_state=randint(0, 1000))
if cofs is None:
cofs = importances.importances_mean
else:
cofs += importances.importances_mean
# analise_auxiliar.find_prediction_time(model, X.shape[1])
# exit(0)
end: float = time.time()
analise_auxiliar.print_time_of_each_prediction(start, end, numpy.size(x_axis),
numpy.size(y))
analise_auxiliar.print_score(numpy.mean(score_test), numpy.mean(score_train))
# print("Using Top {} features: {}".format(args.topk, feature_name[topk_list]))
# bst = xgb.Booster(model_file=args.model_path)
if args.fsel == 1:
skl_model_path = args.model_path.replace("_best.model", "_best.skl")
reg0 = pickle.load(open(skl_model_path, 'rb'))
print("topk important_fea")
topk_list = get_topk_important_fea1(reg0, args.topk)
elif args.fsel == 2:
skl_model_path = args.model_path.replace("_best.model", "_best.skl")
reg0 = pickle.load(open(skl_model_path, 'rb'))
print("topk permutation_importance")
result = permutation_importance(reg0,
e_x,
e_y,
n_repeats=10,
random_state=42,
n_jobs=2)
topk_list = result.importances_mean.argsort()[::-1][:
args.topk] #[::-1]
else:
print("topk gain")
bst = xgb.Booster(model_file=args.model_path)
topk_list = get_topk_important_fea2(bst, args.topk)
feature_list = feature_name[topk_list]
t_x = t_x[:, topk_list]
e_x = e_x[:, topk_list]
if preddata != "":
p_x = p_x[:, topk_list]
print("Using Top {} features: {}".format(args.topk, topk_list))
def test_permutation_importance_equivalence_array_dataframe(n_jobs, max_samples):
# This test checks that the column shuffling logic has the same behavior
# both a dataframe and a simple numpy array.
pd = pytest.importorskip("pandas")
# regression test to make sure that sequential and parallel calls will
# output the same results.
X, y = make_regression(n_samples=100, n_features=5, random_state=0)
X_df = pd.DataFrame(X)
# Add a categorical feature that is statistically linked to y:
binner = KBinsDiscretizer(n_bins=3, encode="ordinal")
cat_column = binner.fit_transform(y.reshape(-1, 1))
# Concatenate the extra column to the numpy array: integers will be
# cast to float values
X = np.hstack([X, cat_column])
assert X.dtype.kind == "f"
# Insert extra column as a non-numpy-native dtype (while keeping backward
# compat for old pandas versions):
if hasattr(pd, "Categorical"):
cat_column = pd.Categorical(cat_column.ravel())
else:
cat_column = cat_column.ravel()
new_col_idx = len(X_df.columns)
X_df[new_col_idx] = cat_column
assert X_df[new_col_idx].dtype == cat_column.dtype
# Stich an aribtrary index to the dataframe:
X_df.index = np.arange(len(X_df)).astype(str)
rf = RandomForestRegressor(n_estimators=5, max_depth=3, random_state=0)
rf.fit(X, y)
n_repeats = 3
importance_array = permutation_importance(
rf,
X,
y,
n_repeats=n_repeats,
random_state=0,
n_jobs=n_jobs,
max_samples=max_samples,
)
# First check that the problem is structured enough and that the model is
# complex enough to not yield trivial, constant importances:
imp_min = importance_array["importances"].min()
imp_max = importance_array["importances"].max()
assert imp_max - imp_min > 0.3
# Now check that importances computed on dataframe matche the values
# of those computed on the array with the same data.
importance_dataframe = permutation_importance(
rf,
X_df,
y,
n_repeats=n_repeats,
random_state=0,
n_jobs=n_jobs,
max_samples=max_samples,
)
assert_allclose(
importance_array["importances"], importance_dataframe["importances"]
)
def test_permutation_importance_sample_weight():
# Creating data with 2 features and 1000 samples, where the target
# variable is a linear combination of the two features, such that
# in half of the samples the impact of feature 1 is twice the impact of
# feature 2, and vice versa on the other half of the samples.
rng = np.random.RandomState(1)
n_samples = 1000
n_features = 2
n_half_samples = n_samples // 2
x = rng.normal(0.0, 0.001, (n_samples, n_features))
y = np.zeros(n_samples)
y[:n_half_samples] = 2 * x[:n_half_samples, 0] + x[:n_half_samples, 1]
y[n_half_samples:] = x[n_half_samples:, 0] + 2 * x[n_half_samples:, 1]
# Fitting linear regression with perfect prediction
lr = LinearRegression(fit_intercept=False)
lr.fit(x, y)
# When all samples are weighted with the same weights, the ratio of
# the two features importance should equal to 1 on expectation (when using
# mean absolutes error as the loss function).
pi = permutation_importance(
lr, x, y, random_state=1, scoring="neg_mean_absolute_error", n_repeats=200
)
x1_x2_imp_ratio_w_none = pi.importances_mean[0] / pi.importances_mean[1]
assert x1_x2_imp_ratio_w_none == pytest.approx(1, 0.01)
# When passing a vector of ones as the sample_weight, results should be
# the same as in the case that sample_weight=None.
w = np.ones(n_samples)
pi = permutation_importance(
lr,
x,
y,
random_state=1,
scoring="neg_mean_absolute_error",
n_repeats=200,
sample_weight=w,
)
x1_x2_imp_ratio_w_ones = pi.importances_mean[0] / pi.importances_mean[1]
assert x1_x2_imp_ratio_w_ones == pytest.approx(x1_x2_imp_ratio_w_none, 0.01)
# When the ratio between the weights of the first half of the samples and
# the second half of the samples approaches to infinity, the ratio of
# the two features importance should equal to 2 on expectation (when using
# mean absolutes error as the loss function).
w = np.hstack(
[np.repeat(10.0 ** 10, n_half_samples), np.repeat(1.0, n_half_samples)]
)
lr.fit(x, y, w)
pi = permutation_importance(
lr,
x,
y,
random_state=1,
scoring="neg_mean_absolute_error",
n_repeats=200,
sample_weight=w,
)
x1_x2_imp_ratio_w = pi.importances_mean[0] / pi.importances_mean[1]
assert x1_x2_imp_ratio_w / x1_x2_imp_ratio_w_none == pytest.approx(2, 0.01)
def test_robustness_to_high_cardinality_noisy_feature(n_jobs, max_samples, seed=42):
# Permutation variable importance should not be affected by the high
# cardinality bias of traditional feature importances, especially when
# computed on a held-out test set:
rng = np.random.RandomState(seed)
n_repeats = 5
n_samples = 1000
n_classes = 5
n_informative_features = 2
n_noise_features = 1
n_features = n_informative_features + n_noise_features
# Generate a multiclass classification dataset and a set of informative
# binary features that can be used to predict some classes of y exactly
# while leaving some classes unexplained to make the problem harder.
classes = np.arange(n_classes)
y = rng.choice(classes, size=n_samples)
X = np.hstack([(y == c).reshape(-1, 1) for c in classes[:n_informative_features]])
X = X.astype(np.float32)
# Not all target classes are explained by the binary class indicator
# features:
assert n_informative_features < n_classes
# Add 10 other noisy features with high cardinality (numerical) values
# that can be used to overfit the training data.
X = np.concatenate([X, rng.randn(n_samples, n_noise_features)], axis=1)
assert X.shape == (n_samples, n_features)
# Split the dataset to be able to evaluate on a held-out test set. The
# Test size should be large enough for importance measurements to be
# stable:
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.5, random_state=rng
)
clf = RandomForestClassifier(n_estimators=5, random_state=rng)
clf.fit(X_train, y_train)
# Variable importances computed by impurity decrease on the tree node
# splits often use the noisy features in splits. This can give misleading
# impression that high cardinality noisy variables are the most important:
tree_importances = clf.feature_importances_
informative_tree_importances = tree_importances[:n_informative_features]
noisy_tree_importances = tree_importances[n_informative_features:]
assert informative_tree_importances.max() < noisy_tree_importances.min()
# Let's check that permutation-based feature importances do not have this
# problem.
r = permutation_importance(
clf,
X_test,
y_test,
n_repeats=n_repeats,
random_state=rng,
n_jobs=n_jobs,
max_samples=max_samples,
)
assert r.importances.shape == (X.shape[1], n_repeats)
# Split the importances between informative and noisy features
informative_importances = r.importances_mean[:n_informative_features]
noisy_importances = r.importances_mean[n_informative_features:]
# Because we do not have a binary variable explaining each target classes,
# the RF model will have to use the random variable to make some
# (overfitting) splits (as max_depth is not set). Therefore the noisy
# variables will be non-zero but with small values oscillating around
# zero:
assert max(np.abs(noisy_importances)) > 1e-7
assert noisy_importances.max() < 0.05
# The binary features correlated with y should have a higher importance
# than the high cardinality noisy features.
# The maximum test accuracy is 2 / 5 == 0.4, each informative feature
# contributing approximately a bit more than 0.2 of accuracy.
assert informative_importances.min() > 0.15
xTreino = xTreino.values
yTreino = yTreino.values
#Define o classificador
classifier = RandomForestClassifier(class_weight="balanced", random_state=1986)
#Treina com todos registros
classifier.fit(xTreino, yTreino)
#Define o scoring
scoring = ['accuracy', 'balanced_accuracy', 'average_precision', 'recall', 'jaccard']
score = 'average_precision'
#Permutation Importance
print('\nPermutation Importance')
pi = permutation_importance(classifier, x, y, scoring=score, n_jobs=3, random_state=1986)
#Restringe as features
indFeatures = np.where((pi.importances_mean * 1000) >= 0.001)[0]
for i in pi.importances_mean[indFeatures].argsort()[::-1]:
print('%s: %.2f' % (features[indFeatures[i]], pi.importances_mean[indFeatures[i]] * 1000))
xTreino = xTreino[:, indFeatures]
xTeste = xTeste[xTeste.columns[indFeatures]]
print('Qtde features selecionadas: ', len(xTeste.columns))
#K-fold
print('\n========== TUNING PARAMETERS ==========')
arrayYReal = []
arrayYPrediction = []
arrayAcuracia = []
average_precisions_train[i] = average_precision_score(
clss_test, pred_train)
average_precisions_test[i] = average_precision_score(clss_test, pred_test)
precision_train, recall_train, _ = precision_recall_curve(
clss_train, pred_train)
precisions_train[i] = np.interp(interval, recall_train[::-1],
precision_train[::-1])
precision_test, recall_test, _ = precision_recall_curve(
clss_test, pred_test)
precisions_test[i] = np.interp(interval, recall_test[::-1],
precision_test[::-1])
selected_features_mask = (classifier.best_estimator_.
named_steps['feature_selection'].get_support())
feat_imp = permutation_importance(
classifier.best_estimator_.named_steps['classify'],
features.values[:, selected_features_mask],
clss,
n_jobs=-1,
random_state=42)
feature_importances[i, selected_features_mask] += \
feat_imp.importances_mean
best_params.append(
(classifier.best_estimator_.named_steps['feature_selection'].k,
classifier.best_estimator_.named_steps['classify'].max_depth,
classifier.best_estimator_.named_steps['classify'].min_samples_leaf))
print(best_params)
stats = {
'accuracy_train': np.mean(accuracies_train),
'accuracy_std_train': np.std(accuracies_train),
'average_precision_train': np.mean(average_precisions_train),
###########################
# IMPORTANCES SORTED #
###########################
importance=modelos["Tree"].feature_importances_
Tree_Sorted={}
for i,v in enumerate(importance):
Tree_Sorted[Features_usados[i]]=np.abs(v)
importance=modelos["Random"].feature_importances_
Random_Sorted={}
for i,v in enumerate(importance):
Random_Sorted[Features_usados[i]]=np.abs(v)
results = permutation_importance(modelos["Knbrs"], X_train, y_train, scoring='accuracy')
importance = results.importances_mean
Knbrs_Sorted={}
for i,v in enumerate(importance):
Knbrs_Sorted[Features_usados[i]]=np.abs(v)
Vector_Sorted={}
importance = modelos["Vector"].coef_[0]
for i,v in enumerate(importance):
Vector_Sorted[Features_usados[i]]=np.abs(v)
Tree_Sorted={k: v for k, v in sorted(Tree_Sorted.items(), key=lambda item: item[1])}
Random_Sorted={k: v for k, v in sorted(Random_Sorted.items(), key=lambda item: item[1])}
Knbrs_Sorted={k: v for k, v in sorted(Knbrs_Sorted.items(), key=lambda item: item[1])}
Vector_Sorted={k: v for k, v in sorted(Vector_Sorted.items(), key=lambda item: item[1])}
Sorteados={
##############################################################################
# NEURAL NETWORK MODEL
##############################################################################
from sklearn.neural_network import MLPClassifier
X_train, X_test, y_train, y_test = train_test_split(features[feature_names],
labels, train_size = 0.9,
random_state = 42,
stratify = labels)
t0 = time.time()
# ???????????????????
nnet = clf = MLPClassifier(solver='lbfgs', alpha=1e-5,
hidden_layer_sizes=(15,), random_state=1)
nnet.fit(X_train, y_train)
t1 = time.time()
total_time = t1-t0 random_state = 42,
stratify = labels)
result = permutation_importance(nnet, X_train, y_train, random_state = 8)
predictions = nnet.predict(X_test)
accuracy = accuracy_score(y_test, predictions)
confmat = confusion_matrix(y_test, predictions)
df_confmat = pd.DataFrame(confmat)
plot_confusion_matrix(nnet, X_test, y_test)
102: 'random_1',
103: 'random_10',
104: 'random_100',
105: 'random_1000',
106: 'random_10000',
107: 'random_100000'
}
normal_games()
clf_tree = RandomForestClassifier(n_estimators=10)
best_features = dict()
clf_tree.fit(games, results)
start = time.time()
result = permutation_importance(clf_tree,
games,
results,
n_repeats=2000,
random_state=0)
print(time.time() - start)
best = result.importances_mean
#best = clf.feature_importances_
for i in range(len(features_dict)):
best_features[features_dict[i]] = best[i]
sort_orders = sorted(best_features.items(), key=lambda x: x[1], reverse=True)
for i in sort_orders:
print(i[0], i[1])
clf_rand_100 = RandomForestClassifier()
# print("**************************************************")
plt.xlabel('LR experiment [mm]')
plt.ylabel('LR predict [mm]')
plt.title("mlp")
#plt.xticks(np.arange(6, 11, step=2))
#plt.yticks(np.arange(6, 11, step=2))
plt.tight_layout()
plt.savefig("gurafu5(研究報告).png")
#3.4特徴量の寄与度
#順列重要度vs特徴量
result = permutation_importance(grid1,
sX_trainH2,
y_trainH2,
n_repeats=5,
random_state=42)
cols = list(sX_trainH2.columns) # 特徴量名のリスト(目的変数CRIM以外)
f_importance = np.array(result["importances"].mean(axis=1)) # 特徴量重要度の算出
f_importance = f_importance / np.sum(f_importance) # 正規化(必要ない場合はコメントアウト)
df_importance = pd.DataFrame({'feature': cols, 'importance': f_importance})
df1 = df_importance
df_importance = df_importance.sort_values("importance", ascending=False)
plt.figure(figsize=(8, 8))
plt.rcParams['font.size'] = 18
plt.rcParams['font.family'] = 'Arial'
plt.rcParams['lines.linewidth'] = 2
plt.savefig("gurafu4.png")
#3.4特徴量の寄与度
#順列重要度vs特徴量
result = permutation_importance(grid,X_train,y_train, n_repeats=5, random_state=42)
df_importance = pd.DataFrame(zip(X_train.columns, result["importances"].mean(axis=1)),columns=["feature","importance"])
df_importance = df_importance.sort_values("importance",ascending=False)
print(df_importance)
df_importance.to_excel('/mnt/c/CEA/df_importance1.xlsx')
plt.figure(figsize=(8,8))
plt.rcParams['font.size'] = 18
plt.rcParams['font.family'] = 'Arial'
plt.rcParams['lines.linewidth'] = 2
plt.rcParams['lines.markersize'] = 4.0
def kfold_cv_LGBMClassifier(
self,
lgb_params: dict,
df: pd.DataFrame,
num_folds: int,
target_col: str,
del_cols=None,
select_cols=None,
eval_metric="error",
stratified=True, # StratifiedKFoldにするか
is_submission=False, # Home_Credit_Default_Risk の submission.csv作成するか
is_plot_perm_importance=False, # permutation importanceも出すか. feature_importance はデフォルトでだす
random_state=1001,
):
"""
LGBMClassifierでcross validation + feature_importance/permutation importance plot
"""
# データフレームからID列など不要な列削除
if del_cols is not None:
df = df.drop(del_cols, axis=1)
# 特徴量の列のみ保持
feats = df.columns.to_list()
feats.remove(target_col)
# Divide in training/validation and test data
train_df = df[df[target_col].notnull()].reset_index(drop=True)
test_df = df[df[target_col].isnull()].reset_index(drop=True)
print(
f"INFO: Starting LightGBM. Train shape: {train_df.shape}, test shape: {test_df.shape}"
)
del df
gc.collect()
###################################### cross validation ######################################
# Cross validation model
if stratified:
folds = StratifiedKFold(n_splits=num_folds,
shuffle=True,
random_state=random_state)
else:
folds = KFold(n_splits=num_folds,
shuffle=True,
random_state=random_state)
# Create arrays and dataframes to store results
oof_preds = np.zeros(train_df.shape[0])
sub_preds = np.zeros(test_df.shape[0])
feature_importance_df = pd.DataFrame()
permutation_importance_df = pd.DataFrame()
result_scores = {}
train_probas = {}
test_probas = {}
best_threshold = 0.0
for n_fold, (train_idx, valid_idx) in enumerate(
folds.split(train_df[feats], train_df[target_col])):
print(
f"\n------------------------------------ n_fold={n_fold + 1} ------------------------------------"
)
############################ create fold ############################
fold_df_base = train_df.iloc[train_idx] # エンコディングのベースデータ
v_fold_df = train_df.iloc[valid_idx]
# ターゲットエンコディングとか一括実行
t_fold_df, v_fold_df = self._encoding(fold_df_base, v_fold_df,
target_col)
print(
f"INFO: Encoded Train shape: {t_fold_df.shape}, valid shape: {v_fold_df.shape}"
)
# 指定の列あればそれだけにする
feats = t_fold_df.columns.to_list(
) if select_cols is None else select_cols
if target_col in feats:
feats.remove(target_col)
print(f"INFO: select features: {len(feats)}\n")
train_x, train_y = (
t_fold_df[feats],
t_fold_df[target_col],
)
valid_x, valid_y = (
v_fold_df[feats],
v_fold_df[target_col],
)
############################ train fit ############################
# LightGBM parameters found by Bayesian optimization
clf = LGBMClassifier(**lgb_params)
clf.fit(
train_x,
train_y,
eval_set=[(train_x, train_y), (valid_x, valid_y)],
eval_metric=eval_metric,
verbose=200,
early_stopping_rounds=200,
)
# モデル保存
joblib.dump(clf,
f"{self.output_dir}/lgb-{n_fold + 1}.model",
compress=True)
# モデルのパラメ
pd.DataFrame.from_dict(clf.get_params(), orient="index").to_csv(
f"{self.output_dir}/param.tsv",
sep="\t",
)
############################ valid pred ############################
oof_preds[valid_idx] = clf.predict_proba(
valid_x, num_iteration=clf.best_iteration_)[:, 1]
if eval_metric == "auc":
fold_auc = roc_auc_score(valid_y, oof_preds[valid_idx])
print("\nINFO: Fold %2d AUC : %.6f" % (n_fold + 1, fold_auc))
result_scores[f"fold_auc_{str(n_fold + 1)}"] = fold_auc
elif eval_metric == "error":
# intにしないとaccuracy_score()エラーになる
_pred = oof_preds[valid_idx]
_pred[_pred >= 0.5] = 1
_pred[_pred < 0.5] = 0
fold_err = 1.0 - accuracy_score(valid_y, _pred)
print("\nINFO: Fold %2d error(threshold=0.5) : %.6f" %
(n_fold + 1, fold_err))
result_scores[f"fold_err_{str(n_fold + 1)}"] = fold_err
# best_thresholdで2値化
_pred = oof_preds[valid_idx]
_best_threshold = Model().nelder_mead_th(valid_y, _pred)
_pred[_pred >= _best_threshold] = 1
_pred[_pred < _best_threshold] = 0
fold_err_best_threshold = 1.0 - accuracy_score(valid_y, _pred)
print(
f"\nINFO: Fold %2d error(threshold={_best_threshold}) : %.6f"
% (n_fold + 1, fold_err_best_threshold))
best_threshold += _best_threshold / num_folds
############################ test pred ############################
if test_df.shape[0] > 0:
# ターゲットエンコディングとか一括実行
tr_df, te_df = self._encoding(fold_df_base, test_df,
target_col)
# testの確信度
test_probas[f"fold_{str(n_fold + 1)}"] = clf.predict_proba(
te_df[feats], num_iteration=clf.best_iteration_)[:, 1]
sub_preds += test_probas[
f"fold_{str(n_fold + 1)}"] / folds.n_splits
# 一応trainの確信度も出しておく
train_probas[f"fold_{str(n_fold + 1)}"] = clf.predict_proba(
tr_df[feats], num_iteration=clf.best_iteration_)[:, 1]
############################ importance 計算 ############################
# feature_importance
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat(
[feature_importance_df, fold_importance_df], axis=0)
if is_plot_perm_importance:
# permutation_importance
# 時間かかるからifで制御する
# scoringはsklearnのスコアリングパラメータ
# accuracy や neg_mean_squared_log_error とか
# https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter
fold_importance_df = pd.DataFrame()
fold_permutation = permutation_importance(clf,
valid_x,
valid_y,
scoring="roc_auc")
fold_permutation_df = pd.DataFrame(
{
"feature": valid_x.columns,
"importance":
np.abs(fold_permutation["importances_mean"]
), # マイナスとるのもあるので絶対値にする
"fold": n_fold + 1,
}, )
permutation_importance_df = pd.concat(
[permutation_importance_df, fold_permutation_df], axis=0)
del clf, train_x, train_y, valid_x, valid_y
gc.collect()
print(
"\n------------------------------------ mean fold ------------------------------------"
)
mean_fold_score = None
if eval_metric == "auc":
mean_fold_score = roc_auc_score(train_df[target_col], oof_preds)
print("INFO: Mean valid AUC score %.6f" % mean_fold_score)
result_scores["mean_fold_auc"] = mean_fold_score
elif eval_metric == "error":
# intにしないとaccuracy_score()エラーになる
_pred = oof_preds
_pred[_pred >= 0.5] = 1
_pred[_pred < 0.5] = 0
mean_fold_score = 1.0 - accuracy_score(train_df[target_col], _pred)
print("INFO: Mean valid error score %.6f" % mean_fold_score)
result_scores["mean_fold_err"] = mean_fold_score
# モデルの評価指標出力
result_scores_df = pd.DataFrame(result_scores.values(),
index=result_scores.keys())
result_scores_df.to_csv(f"{self.output_dir}/result_scores.tsv",
sep="\t")
# test setについて
if test_df.shape[0] > 0:
test_probas_df = pd.DataFrame(test_probas)
test_probas_df.to_csv(f"{self.output_dir}/test_probas.csv",
index=False)
############################ Write submission file ############################
if is_submission:
# threshold=0.5で2値化
te_mean = test_probas_df.apply(lambda x: np.mean(x),
axis=1).values
te_mean[te_mean >= 0.5] = 1
te_mean[te_mean < 0.5] = 0
te_mean = te_mean.astype(int)
output_csv = f"{self.output_dir}/submission_kernel.csv"
pd.DataFrame({
"id": range(len(te_mean)),
"y": te_mean
}).to_csv(output_csv, index=False)
print(f"INFO: save csv {output_csv}")
# best_thresholdで2値化
print(
f"INFO: submission best_threshold(cv mean): {best_threshold}"
)
te_mean = test_probas_df.apply(lambda x: np.mean(x),
axis=1).values
te_mean[te_mean >= best_threshold] = 1
te_mean[te_mean < best_threshold] = 0
te_mean = te_mean.astype(int)
output_csv = f"{self.output_dir}/submission_nelder_mead.csv"
pd.DataFrame({
"id": range(len(te_mean)),
"y": te_mean
}).to_csv(output_csv, index=False)
print(f"INFO: save csv {output_csv}")
# Plot feature importance
png_path = f"{self.output_dir}/lgbm_feature_importances.png"
Model().display_importances(
feature_importance_df,
png_path=png_path,
title="feature_importance",
)
# print(f"INFO: save png {png_path}")
if is_plot_perm_importance:
png_path = f"{self.output_dir}/lgbm_permutation_importances.png"
Model().display_importances(
permutation_importance_df,
png_path=png_path,
title="permutation_importance",
)
# print(f"INFO: save png {png_path}")
return mean_fold_score, feature_importance_df, permutation_importance_df
pyplot.show()
# In[27]:
from sklearn.datasets import make_regression
from sklearn.neighbors import KNeighborsRegressor
from sklearn.inspection import permutation_importance
from matplotlib import pyplot
# define dataset
X, y = make_regression(n_samples=1000,
n_features=10,
n_informative=5,
random_state=1)
#define the model
model = KNeighborsRegressor()
#fit thr model
model.fit(X, y)
# perform permutation importance
results = permutation_importance(model, X, y, scoring='neg_mean_squared_error')
#get importance
importance = results.importances_mean
# summarize feature importance
for i, v in enumerate(importance):
print('Feature: %0d, Score: %.5f' % (i, v))
# plot feature importance
pyplot.bar([x for x in range(len(importance))], importance)
pyplot.show()
# In[ ]:
def _get_permutation_importance(self, model, fit_params):
assert fit_params is not None
res = permutation_importance(model, **fit_params)
return res['importances_mean'], res['importances_std']
'critical': y_pool,
'confidence': con,
'confidence_of_1': con_1
}
df_r = pd.DataFrame(data,
columns=[
'Attack-Stage', 'Port-Service', 'critical',
'confidence', 'confidence_of_1'
])
print()
print(df_r.iloc[:, 0:4])
from sklearn.inspection import permutation_importance
r = permutation_importance(learner,
X_pool,
y_pool,
n_repeats=10,
random_state=0)
for i in r.importances_mean.argsort()[::-1]:
print(f"{df.iloc[:,2:7].columns.tolist()[i]:<8}"
f' '
f"{r.importances_mean[i]:.3f}"
f" +/- {r.importances_std[i]:.3f}")
df_A = df_r.groupby(by='Attack-Stage')['confidence_of_1'].mean()
df_P = df_r.groupby(by='Port-Service')['confidence_of_1'].mean()
print()
print(df_A.sort_values())
print()
print(df_P.sort_values())
print()
# for i, v in enumerate(importance):
# print('Feature: %0d, Score: %.5f' % (i, v))
# print('\n')
# except:
# print('Ridge classifier error' + '\n')
# """*******************************************************************************"""
try:
model = MultinomialNB()
model.fit(X_train, y_train)
y_pred = model.predict(X)
print('Result of Multinomial NB')
imps = permutation_importance(model, X_train, y_train)
print(imps.importances_mean)
print('Accurancy score: ' + str(accuracy_score(y, y_pred)))
print('Precision score: ' +
str(precision_score(y, y_pred, average='micro', zero_division=1)))
print('Recall score: ' +
str(recall_score(y, y_pred, average='micro', zero_division=1)))
print('F1 score: ' +
str(f1_score(y, y_pred, average='micro', zero_division=1)) + '\n')
print('Precision score (macro): ' +
str(precision_score(y, y_pred, average='macro', zero_division=1)))
print('Recall score (macro): ' +
str(recall_score(y, y_pred, average='macro', zero_division=1)))
print('F1 score (macro): ' +
str(f1_score(y, y_pred, average='macro', zero_division=1)) + '\n')
# ------------------------------------
#
# The :func:`inspection.permutation_importance` can be used to get an
# estimate of the importance of each feature, for any fitted estimator:
import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.inspection import permutation_importance
X, y = make_classification(random_state=0, n_features=5, n_informative=3)
feature_names = np.array([f'x_{i}' for i in range(X.shape[1])])
rf = RandomForestClassifier(random_state=0).fit(X, y)
result = permutation_importance(rf, X, y, n_repeats=10, random_state=0,
n_jobs=-1)
fig, ax = plt.subplots()
sorted_idx = result.importances_mean.argsort()
ax.boxplot(result.importances[sorted_idx].T,
vert=False, labels=feature_names[sorted_idx])
ax.set_title("Permutation Importance of each feature")
ax.set_ylabel("Features")
fig.tight_layout()
plt.show()
# %%
# Native support for missing values for gradient boosting
# -------------------------------------------------------
#
# The :class:`ensemble.HistGradientBoostingClassifier`
model.fit(data, target) # Training the Classifier.
#@ Evaluation Metrics:
predictions = model.predict(data) # Making the Predictions.
oob = model.oob_score_ # Calculating the OOB score.
oob_list.append(oob)
recall = recall_score(target, predictions) # Calculating the Recall score.
recall_list.append(recall)
kappa = cohen_kappa_score(target, predictions) # Calculating the Kappa score.
kappa_list.append(kappa)
#@ GINI Features Importance:
gini_importances["Run" + str(seed)] = model.feature_importances_ # Features Importance: GINI.
#@ PERMUTATION Features Importance:
permutation_important = permutation_importance(model, data, target, n_repeats=5, # Permutation Importance.
n_jobs=-1, random_state=seed)
permutation_importances["Run" + str(seed)] = permutation_important.importances_mean # Features Importance: PERMUTATION.
# 1. Evaluation Metrics:
# I will calculate the Evaluation metrics such as OOB score, Recall score and Kappa score using the whole dataset.
# I will be changing the seed of Random Forest Classifier along with the loop. The values of seed starts from 123
# and goes up to 148 which will also be the range of Iteration of the loop which is 25 Iterations in total.
#@ Dictionary of Evaluation Metrics:
scores = {"OOB Score": oob_list,
"Recall Score": recall_list,
"Kappa Score": kappa_list}
scores = pd.DataFrame.from_dict(scores, orient="columns") # Creating the DataFrame.
scores.to_csv("./Ecoli_evaluation_metrics_nestimator800.csv") # Saving the DataFrame into csv.
# 2. GINI: Features Importance
ax.set_title("Feature importances")
bp = ax.boxplot(importances,
vert=False,
labels=y_titles[sorted_idx],
showbox=False,
showcaps=False,
showfliers=False,
showmeans=True,
medianprops=dict(color="white"),
whiskerprops=dict(color="white"),
meanprops=dict(color="black"))
sresult = permutation_importance(model,
small_test.drop('phase', axis=1),
small_test.phase,
n_repeats=10,
random_state=42)
mresult = permutation_importance(model,
medium_test.drop('phase', axis=1),
medium_test.phase,
n_repeats=10,
random_state=42)
bresult = permutation_importance(model,
big_test.drop('phase', axis=1),
big_test.phase,
n_repeats=10,
random_state=42)
def test_features_relevance(self,
df,
model_name=None,
model=None,
features=None,
normalize=True,
method=DEFAULT_IMP_METHOD,
target=None,
exclude=None,
positive_only=True):
x, y = [], []
if model is None:
if model_name is not None:
self.activate(model_name)
model = self.model
if model is None:
logging.error(
' Please specify an existing model or train a new one.')
return
if target is None:
target = self.target
if features is None:
features = self.features
features, target = self.get_features(df,
features=features,
exclude=exclude,
target=target)
features_df = pd.DataFrame(columns=features)
if method == 'all':
return pd.DataFrame.from_dict({
m: self.test_features_relevance(df, method=m)
for m in ('coef', 'permutation', 'score', 'residual')
})
if method == 'coef':
if hasattr(model, '_final_estimator'):
estimator = model._final_estimator
else:
estimator = model
if hasattr(estimator, 'coef_'):
importance = estimator.coef_
elif hasattr(estimator, 'feature_importances_'):
importance = estimator.feature_importances_
else:
logging.error(
' Final estimator for specified model has neither "coef_" nor "feature_importances_" attributes, '
'choose another test method.')
return
features_df = pd.Series(dict(zip(
features, abs(importance)))).sort_values(ascending=False)
else:
x = df[features].applymap(partial(pd.to_numeric,
errors='coerce')).values
if method in ('permutation', 'score'):
y = df[target].apply(partial(pd.to_numeric,
errors='coerce')).values
x, y = self.trim_xy(x, y)
else:
x = self.trim_xy(x)
if method == 'permutation':
if hasattr(model, '_final_estimator'):
estimator = model._final_estimator
else:
estimator = model
if hasattr(estimator, '_estimator_type'
) and estimator._estimator_type == 'regressor':
scoring = 'neg_mean_squared_error'
else:
scoring = 'accuracy'
try:
results = permutation_importance(model,
x,
y,
scoring=scoring)
importance = results.importances_mean
except ValueError:
importance = pd.Series([np.nan] * len(features))
features_df = pd.Series(dict(zip(features, abs(importance))))
elif method == 'residual':
for row_index, row_feats in enumerate(tqdm(x)):
features_relevance = {}
baseline_prediction = model.predict([row_feats])
for index, feature_value in enumerate(row_feats):
new_row = row_feats.copy()
new_row[
index] = -1 / feature_value if feature_value > 1e-6 else 1e6
new_prediction = model.predict([new_row])
features_relevance.update({
index:
abs(new_prediction - baseline_prediction)[0]
})
features_df = features_df.append(dict(
zip(features_df.columns, features_relevance.values())),
ignore_index=True)
# noinspection PyArgumentList
features_df = features_df.sum()
elif method == 'score':
baseline_score = model.score(x, y)
for index, feature in enumerate(features_df.columns):
new_x = x.copy()
new_x[:, index] = x[:, index].mean()
features_df.loc[0, feature] = (
baseline_score / model.score(new_x, y) - 1) * 100
features_df = features_df.T[0]
else:
logging.error(
' Valid methods are "coef", "permutation", "residual" and "score".'
)
return
if positive_only:
features_df = features_df[features_df > 0]
if normalize:
min_max_scaler = MinMaxScaler()
features_df = pd.Series((min_max_scaler.fit_transform(
(lambda values: values.reshape([len(values), 1]))(
features_df.values)) * 100).flatten(), features_df.keys())
features_df = features_df.sort_values(ascending=False)
return features_df
ax.set_yticks(y_ticks)
ax.set_yticklabels(feature_names[sorted_idx])
ax.set_title("Random Forest Feature Importances (MDI)")
fig.tight_layout()
plt.show()
# %%
# As an alternative, the permutation importances of ``rf`` are computed on a
# held out test set. This shows that the low cardinality categorical feature,
# ``sex`` is the most important feature.
#
# Also note that both random features have very low importances (close to 0) as
# expected.
result = permutation_importance(rf,
X_test,
y_test,
n_repeats=10,
random_state=42,
n_jobs=2)
sorted_idx = result.importances_mean.argsort()
fig, ax = plt.subplots()
ax.boxplot(result.importances[sorted_idx].T,
vert=False,
labels=X_test.columns[sorted_idx])
ax.set_title("Permutation Importances (test set)")
fig.tight_layout()
plt.show()
# %%
# It is also possible to compute the permutation importances on the training
# set. This reveals that ``random_num`` gets a significantly higher importance
def generate_permutation_importance(self):
""" 11. Generate permutation importance. """
self.permutation_importance = permutation_importance(
self.mdl_(), self.X_train, self.X_ttest)
self.next(self.end)