def fit(self, X: pd.DataFrame, y: pd.Series):
"""
Find the important features. Note that the selector trains various models at
each round of selection, so it might take a while.
Parameters
----------
X : pandas dataframe of shape = [n_samples, n_features]
The input dataframe
y : array-like of shape (n_samples)
Target variable. Required to train the estimator.
Returns
-------
self
"""
# check input dataframe
X = _is_dataframe(X)
# find numerical variables or check variables entered by user
self.variables = _find_or_check_numerical_variables(X, self.variables)
# train model with all features and cross-validation
model = cross_validate(
self.estimator,
X[self.variables],
y,
cv=self.cv,
scoring=self.scoring,
return_estimator=True,
)
# store initial model performance
self.initial_model_performance_ = model["test_score"].mean()
# Initialize a dataframe that will contain the list of the feature/coeff
# importance for each cross validation fold
feature_importances_cv = pd.DataFrame()
# Populate the feature_importances_cv dataframe with columns containing
# the feature importance values for each model returned by the cross
# validation.
# There are as many columns as folds.
for m in model["estimator"]:
feature_importances_cv[m] = get_feature_importances(m)
# Add the variables as index to feature_importances_cv
feature_importances_cv.index = self.variables
# Aggregate the feature importance returned in each fold
self.feature_importances_ = feature_importances_cv.mean(axis=1)
# Sort the feature importance values decreasingly
self.feature_importances_.sort_values(ascending=False, inplace=True)
# Extract most important feature from the ordered list of features
first_most_important_feature = list(self.feature_importances_.index)[0]
# Run baseline model using only the most important feature
baseline_model = cross_validate(
self.estimator,
X[first_most_important_feature].to_frame(),
y,
cv=self.cv,
scoring=self.scoring,
return_estimator=True,
)
# Save baseline model performance
baseline_model_performance = baseline_model["test_score"].mean()
# list to collect selected features
# It is initialized with the most important feature
_selected_features = [first_most_important_feature]
# dict to collect features and their performance_drift
# It is initialized with the performance drift of
# the most important feature
self.performance_drifts_ = {first_most_important_feature: 0}
# loop over the ordered list of features by feature importance starting
# from the second element in the list.
for feature in list(self.feature_importances_.index)[1:]:
# Add feature and train new model
model_tmp = cross_validate(
self.estimator,
X[_selected_features + [feature]],
y,
cv=self.cv,
scoring=self.scoring,
return_estimator=True,
)
# assign new model performance
model_tmp_performance = model_tmp["test_score"].mean()
# Calculate performance drift
performance_drift = model_tmp_performance - baseline_model_performance
# Save feature and performance drift
self.performance_drifts_[feature] = performance_drift
# If new performance model is
if performance_drift > self.threshold:
# add feature to the list of selected features
_selected_features.append(feature)
# Update new baseline model performance
baseline_model_performance = model_tmp_performance
self.features_to_drop_ = [
f for f in self.variables if f not in _selected_features
]
self.input_shape_ = X.shape
return self
def fit(self, X, y):
"""
Args
----
X: pandas dataframe of shape = [n_samples, n_features]
The input dataframe
y: array-like of shape (n_samples)
Target variable. Required to train the estimator.
Returns
-------
self
"""
# check input dataframe
X = _is_dataframe(X)
# find numerical variables or check variables entered by user
self.variables = _find_numerical_variables(X, self.variables)
# train model with all features and cross-validation
model = cross_validate(
self.estimator,
X[self.variables],
y,
cv=self.cv,
scoring=self.scoring,
return_estimator=True,
)
# store initial model performance
self.initial_model_performance_ = model["test_score"].mean()
# Initialize a dataframe that will contain the list of the feature/coeff
# importance for each cross validation fold
feature_importances_cv = pd.DataFrame()
# Populate the feature_importances_cv dataframe with columns containing
# the feature importance values for each model returned by the cross
# validation.
# There are as many columns as folds.
for m in model["estimator"]:
feature_importances_cv[m] = get_feature_importances(m)
# Add the variables as index to feature_importances_cv
feature_importances_cv.index = self.variables
# Aggregate the feature importance returned in each fold
self.feature_importances_ = feature_importances_cv.mean(axis=1)
# Sort the feature importance values
self.feature_importances_.sort_values(ascending=True, inplace=True)
# list to collect selected features
self.selected_features_ = []
# temporary copy where we will remove features recursively
X_tmp = X[self.variables].copy()
# we need to update the performance as we remove features
baseline_model_performance = self.initial_model_performance_
# dict to collect features and their performance_drift after shuffling
self.performance_drifts_ = {}
# evaluate every feature, starting from the least important
# remember that feature_importances_ is ordered already
for feature in list(self.feature_importances_.index):
# remove feature and train new model
model_tmp = cross_validate(
self.estimator,
X_tmp.drop(columns=feature),
y,
cv=self.cv,
scoring=self.scoring,
return_estimator=False,
)
# assign new model performance
model_tmp_performance = model_tmp["test_score"].mean()
# Calculate performance drift
performance_drift = baseline_model_performance - model_tmp_performance
# Save feature and performance drift
self.performance_drifts_[feature] = performance_drift
if performance_drift > self.threshold:
self.selected_features_.append(feature)
else:
# remove feature and adjust initial performance
X_tmp = X_tmp.drop(columns=feature)
baseline_model = cross_validate(
self.estimator,
X_tmp,
y,
cv=self.cv,
return_estimator=False,
scoring=self.scoring,
)
# store initial model performance
baseline_model_performance = baseline_model["test_score"].mean(
)
self.input_shape_ = X.shape
return self
def fit(self, X: pd.DataFrame, y: pd.Series):
"""
Find initial model performance. Sort features by importance.
Parameters
----------
X: pandas dataframe of shape = [n_samples, n_features]
The input dataframe
y: array-like of shape (n_samples)
Target variable. Required to train the estimator.
"""
# check input dataframe
X, y = check_X_y(X, y)
# If required exclude variables that are not in the input dataframe
self._confirm_variables(X)
# find numerical variables or check variables entered by user
self.variables_ = _find_or_check_numerical_variables(
X, self.variables_)
# check that there are more than 1 variable to select from
self._check_variable_number()
# save input features
self._get_feature_names_in(X)
# train model with all features and cross-validation
model = cross_validate(
self.estimator,
X[self.variables_],
y,
cv=self.cv,
scoring=self.scoring,
return_estimator=True,
)
# store initial model performance
self.initial_model_performance_ = model["test_score"].mean()
# Initialize a dataframe that will contain the list of the feature/coeff
# importance for each cross validation fold
feature_importances_cv = pd.DataFrame()
# Populate the feature_importances_cv dataframe with columns containing
# the feature importance values for each model returned by the cross
# validation.
# There are as many columns as folds.
for i in range(len(model["estimator"])):
m = model["estimator"][i]
feature_importances_cv[i] = get_feature_importances(m)
# Add the variables as index to feature_importances_cv
feature_importances_cv.index = self.variables_
# Aggregate the feature importance returned in each fold
self.feature_importances_ = feature_importances_cv.mean(axis=1)
return X, y