def fit(self, X: pd.DataFrame, y: pd.Series): """ Find the important features. Note that the selector trains various models at each round of selection, so it might take a while. Parameters ---------- X : pandas dataframe of shape = [n_samples, n_features] The input dataframe y : array-like of shape (n_samples) Target variable. Required to train the estimator. Returns ------- self """ # check input dataframe X = _is_dataframe(X) # find numerical variables or check variables entered by user self.variables = _find_or_check_numerical_variables(X, self.variables) # train model with all features and cross-validation model = cross_validate( self.estimator, X[self.variables], y, cv=self.cv, scoring=self.scoring, return_estimator=True, ) # store initial model performance self.initial_model_performance_ = model["test_score"].mean() # Initialize a dataframe that will contain the list of the feature/coeff # importance for each cross validation fold feature_importances_cv = pd.DataFrame() # Populate the feature_importances_cv dataframe with columns containing # the feature importance values for each model returned by the cross # validation. # There are as many columns as folds. for m in model["estimator"]: feature_importances_cv[m] = get_feature_importances(m) # Add the variables as index to feature_importances_cv feature_importances_cv.index = self.variables # Aggregate the feature importance returned in each fold self.feature_importances_ = feature_importances_cv.mean(axis=1) # Sort the feature importance values decreasingly self.feature_importances_.sort_values(ascending=False, inplace=True) # Extract most important feature from the ordered list of features first_most_important_feature = list(self.feature_importances_.index)[0] # Run baseline model using only the most important feature baseline_model = cross_validate( self.estimator, X[first_most_important_feature].to_frame(), y, cv=self.cv, scoring=self.scoring, return_estimator=True, ) # Save baseline model performance baseline_model_performance = baseline_model["test_score"].mean() # list to collect selected features # It is initialized with the most important feature _selected_features = [first_most_important_feature] # dict to collect features and their performance_drift # It is initialized with the performance drift of # the most important feature self.performance_drifts_ = {first_most_important_feature: 0} # loop over the ordered list of features by feature importance starting # from the second element in the list. for feature in list(self.feature_importances_.index)[1:]: # Add feature and train new model model_tmp = cross_validate( self.estimator, X[_selected_features + [feature]], y, cv=self.cv, scoring=self.scoring, return_estimator=True, ) # assign new model performance model_tmp_performance = model_tmp["test_score"].mean() # Calculate performance drift performance_drift = model_tmp_performance - baseline_model_performance # Save feature and performance drift self.performance_drifts_[feature] = performance_drift # If new performance model is if performance_drift > self.threshold: # add feature to the list of selected features _selected_features.append(feature) # Update new baseline model performance baseline_model_performance = model_tmp_performance self.features_to_drop_ = [ f for f in self.variables if f not in _selected_features ] self.input_shape_ = X.shape return self
def fit(self, X, y): """ Args ---- X: pandas dataframe of shape = [n_samples, n_features] The input dataframe y: array-like of shape (n_samples) Target variable. Required to train the estimator. Returns ------- self """ # check input dataframe X = _is_dataframe(X) # find numerical variables or check variables entered by user self.variables = _find_numerical_variables(X, self.variables) # train model with all features and cross-validation model = cross_validate( self.estimator, X[self.variables], y, cv=self.cv, scoring=self.scoring, return_estimator=True, ) # store initial model performance self.initial_model_performance_ = model["test_score"].mean() # Initialize a dataframe that will contain the list of the feature/coeff # importance for each cross validation fold feature_importances_cv = pd.DataFrame() # Populate the feature_importances_cv dataframe with columns containing # the feature importance values for each model returned by the cross # validation. # There are as many columns as folds. for m in model["estimator"]: feature_importances_cv[m] = get_feature_importances(m) # Add the variables as index to feature_importances_cv feature_importances_cv.index = self.variables # Aggregate the feature importance returned in each fold self.feature_importances_ = feature_importances_cv.mean(axis=1) # Sort the feature importance values self.feature_importances_.sort_values(ascending=True, inplace=True) # list to collect selected features self.selected_features_ = [] # temporary copy where we will remove features recursively X_tmp = X[self.variables].copy() # we need to update the performance as we remove features baseline_model_performance = self.initial_model_performance_ # dict to collect features and their performance_drift after shuffling self.performance_drifts_ = {} # evaluate every feature, starting from the least important # remember that feature_importances_ is ordered already for feature in list(self.feature_importances_.index): # remove feature and train new model model_tmp = cross_validate( self.estimator, X_tmp.drop(columns=feature), y, cv=self.cv, scoring=self.scoring, return_estimator=False, ) # assign new model performance model_tmp_performance = model_tmp["test_score"].mean() # Calculate performance drift performance_drift = baseline_model_performance - model_tmp_performance # Save feature and performance drift self.performance_drifts_[feature] = performance_drift if performance_drift > self.threshold: self.selected_features_.append(feature) else: # remove feature and adjust initial performance X_tmp = X_tmp.drop(columns=feature) baseline_model = cross_validate( self.estimator, X_tmp, y, cv=self.cv, return_estimator=False, scoring=self.scoring, ) # store initial model performance baseline_model_performance = baseline_model["test_score"].mean( ) self.input_shape_ = X.shape return self
def fit(self, X: pd.DataFrame, y: pd.Series): """ Find initial model performance. Sort features by importance. Parameters ---------- X: pandas dataframe of shape = [n_samples, n_features] The input dataframe y: array-like of shape (n_samples) Target variable. Required to train the estimator. """ # check input dataframe X, y = check_X_y(X, y) # If required exclude variables that are not in the input dataframe self._confirm_variables(X) # find numerical variables or check variables entered by user self.variables_ = _find_or_check_numerical_variables( X, self.variables_) # check that there are more than 1 variable to select from self._check_variable_number() # save input features self._get_feature_names_in(X) # train model with all features and cross-validation model = cross_validate( self.estimator, X[self.variables_], y, cv=self.cv, scoring=self.scoring, return_estimator=True, ) # store initial model performance self.initial_model_performance_ = model["test_score"].mean() # Initialize a dataframe that will contain the list of the feature/coeff # importance for each cross validation fold feature_importances_cv = pd.DataFrame() # Populate the feature_importances_cv dataframe with columns containing # the feature importance values for each model returned by the cross # validation. # There are as many columns as folds. for i in range(len(model["estimator"])): m = model["estimator"][i] feature_importances_cv[i] = get_feature_importances(m) # Add the variables as index to feature_importances_cv feature_importances_cv.index = self.variables_ # Aggregate the feature importance returned in each fold self.feature_importances_ = feature_importances_cv.mean(axis=1) return X, y