def permutation_importance(model, x_all, targets_all, config):
_logger.info("Computing permutation importance!!")
if config.algorithm not in transformed_modelmaps.keys():
raise AttributeError("Only the following can be used for permutation "
"importance {}".format(
list(transformed_modelmaps.keys())))
y = targets_all.observations
classification = hasattr(model, 'predict_proba')
if not classification:
for score in ['explained_variance',
'r2',
'neg_mean_absolute_error',
'neg_mean_squared_error']:
pi_cv = apply_multiple_masked(
PermutationImportance(model, scoring=score,
cv='prefit', n_iter=10,
refit=False).fit, data=(x_all, y)
)
feature_names = geoio.feature_names(config)
df_picv = eli5.explain_weights_df(
pi_cv, feature_names=feature_names, top=100)
csv = Path(config.output_dir).joinpath(
config.name + "_permutation_importance_{}.csv".format(
score)).as_posix()
df_picv.to_csv(csv, index=False)
import numpy as np
import pytest
from sklearn.metrics import r2_score
from uncoverml.krige import krige_methods, Krige, all_ml_models, MLKrige
from uncoverml.models import modelmaps
from uncoverml.optimise.models import transformed_modelmaps
models = list(modelmaps.keys()) + list(transformed_modelmaps.keys())
@pytest.fixture(params=[
k for k in models
if k not in ['depthregress', 'cubist', 'multicubist', 'multirandomforest']
])
def get_models(request):
if request.param in modelmaps:
return modelmaps[request.param]
elif request.param in transformed_modelmaps:
return transformed_modelmaps[request.param]
def test_modeltags(get_models):
model = get_models()
tags = model.get_predict_tags()
assert len(tags) >= 1 # at least a predict function
if hasattr(model, 'predict_proba'):
assert len(tags) >= 4 # at least predict, var and upper & lower quant