def feature_selection(self, direction=None): sfs = SequentialFeatureSelector(self.model, n_features_to_select=3, direction=direction) self.sfs = sfs.fit(self.X_train, self.Y_train) self.X_train_columns = self.X_train.columns[sfs.get_support()] self.X_test_columns = self.X_test.columns[sfs.get_support()] self.X_train = self.X_train[self.X_train_columns] self.X_test = self.X_test[self.X_test_columns] self.model = LinearRegression().fit(self.X_train, self.Y_train)
def run_sklearn(self): """ Train and evaluate models from sklearn """ classifier_options = { #'lr': linear_model.LogisticRegressionCV(cv=10, n_jobs=10, max_iter=10000, verbose=0), 'lr': linear_model.LogisticRegression(n_jobs=10, max_iter=100000, verbose=0), 'svm': model_selection.GridSearchCV(svm.LinearSVC(dual=False, max_iter=10000, verbose=0), { 'C': [.01, .1, 1, 10, 100], 'penalty': ['l2'] }, n_jobs=10, cv=10, verbose=2), 'mlp': neural_network.MLPClassifier(hidden_layer_sizes=(32, 50), activation='relu', early_stopping=True, verbose=2) } self.clf = classifier_options[self.clf_type] if self.sfs_k > 0: # Forward feature selection print("Doing forward feature selection...") sfs = SequentialFeatureSelector(self.clf, n_features_to_select=self.sfs_k, n_jobs=-1) sfs.fit(self.data.X_train, self.data.y_train) # Save out selected features outpath = os.path.join( '/projects/tumblr_community_identity/tmp/', f'sfs{self.sfs_k}_{self.extractor.select_k}.txt') np.savetxt(outpath, sfs.get_support()) print(f"Saved forward feature selection mask to {outpath}") sfs_mask = sfs.get_support() #sfs_mask = np.loadtxt( # '/projects/tumblr_community_identity/tmp/sfs20_500.txt').astype(bool) X_train = self.data.X_train[:, sfs_mask] X_dev = self.data.X_dev[:, sfs_mask] X_test = self.data.X_test[:, sfs_mask] self.data.X_train, self.data.X_dev, self.data.X_test = X_train, X_dev, X_test self.model = self.clf.fit(self.data.X_train, self.data.y_train) self.test_score = self.model.score(self.data.X_test, self.data.y_test) self.train_pred = self.model.predict(self.data.X_train) if self.data.X_dev is not None: self.dev_score = self.model.score(self.data.X_dev, self.data.y_dev) self.dev_pred = self.model.predict(self.data.X_dev) self.test_pred = self.model.predict(self.data.X_test)
def seq_feature_selection(data, target, n_features=None): predictors = data.drop(columns=target).select_dtypes(np.number) selector = SequentialFeatureSelector(estimator=LinearRegression(), n_features_to_select=n_features) selector = selector.fit(predictors, data[target]) selected = selector.get_support(indices=True) return predictors.iloc[:, selected].columns.to_list()
def test_n_features_to_select(direction, n_features_to_select): # Make sure n_features_to_select is respected X, y = make_regression(n_features=10) sfs = SequentialFeatureSelector(LinearRegression(), n_features_to_select=n_features_to_select, direction=direction, cv=2) sfs.fit(X, y) if n_features_to_select is None: n_features_to_select = 5 # n_features // 2 assert sfs.get_support(indices=True).shape[0] == n_features_to_select assert sfs.n_features_to_select_ == n_features_to_select assert sfs.transform(X).shape[1] == n_features_to_select
def test_n_features_to_select_auto(direction): """Check the behaviour of `n_features_to_select="auto"` with different values for the parameter `tol`. """ n_features = 10 tol = 1e-3 X, y = make_regression(n_features=n_features, random_state=0) sfs = SequentialFeatureSelector( LinearRegression(), n_features_to_select="auto", tol=tol, direction=direction, cv=2, ) sfs.fit(X, y) max_features_to_select = n_features - 1 assert sfs.get_support(indices=True).shape[0] <= max_features_to_select assert sfs.n_features_to_select_ <= max_features_to_select assert sfs.transform(X).shape[1] <= max_features_to_select assert sfs.get_support(indices=True).shape[0] == sfs.n_features_to_select_
def select_greedy(data): X, X_test, y = data svr = svm.SVR(kernel="rbf", C=100, tol=1).fit(X, y) tic = time() select = SequentialFeatureSelector(svr, direction=direction, n_features_to_select=n_features, n_jobs=-1).fit(X, y) toc = time() joblib.dump(select.get_support(), "joblib/greedy_support") print(f"features selected: {select.get_support()}") print(f"done in: {toc - tic:.2f}s") return select.transform(X), select.transform(X_test), y
def test_sanity(seed, direction, n_features_to_select, expected_selected_features): # Basic sanity check: 3 features, only f0 and f2 are correlated with the # target, f2 having a stronger correlation than f0. We expect f1 to be # dropped, and f2 to always be selected. rng = np.random.RandomState(seed) n_samples = 100 X = rng.randn(n_samples, 3) y = 3 * X[:, 0] - 10 * X[:, 2] sfs = SequentialFeatureSelector(LinearRegression(), n_features_to_select=n_features_to_select, direction=direction, cv=2) sfs.fit(X, y) assert_array_equal(sfs.get_support(indices=True), expected_selected_features)
from sklearn.feature_selection import SequentialFeatureSelector model = LinearSVC(C=0.01, penalty='l2', dual=False, random_state=0).fit(x, y) #C越小惩罚力度越大 names = np.array(columns) sfs_forward = SequentialFeatureSelector(model, n_features_to_select=7, scoring='accuracy', cv=5).fit(x, y) sfs_backward = SequentialFeatureSelector(model, n_features_to_select=7, scoring='accuracy', direction='backward', cv=5).fit(x, y) print('Features selected by forward SFS selection l2:', names[sfs_forward.get_support()]) print('Features selected by backward SFS selection l2:', names[sfs_backward.get_support()]) ''' L1惩罚项降维的原理在于保留多个对目标值具有同等相关性的特征中的一个,所以没选到的特征不代表不重要。 故,可结合L2惩罚项来优化。具体操作为:若一个特征在L1中的权值为1,选择在L2中权值差别不大且在L1中权值为0的特征构成同类集合, 将这一集合中的特征平分L1中的权值,故需要构建一个新的模型: ''' class LS(LinearSVC): def __init__(self, threshold=0.01, dual=False, tol=1e-4, C=0.01,
valid_df[valid_df.columns[i]]) train_df[train_df.columns[i]] = clean_Dirt_Data( train_df[train_df.columns[i]]) x_train, y_train = prepareData(train_df) x_valid, y_valid = prepareData(valid_df) print('Done Read Train and Validation data!') #Avaliacao do NB classificador_nb = GaussianNB(priors=None, var_smoothing=1e-9) # Sequential Forward Selection(sfs) sfs = SequentialFeatureSelector(classificador_nb, n_features_to_select=x_train.shape[1] * 0.15) sfs.fit(x_train, y_train) result_nb = sfs.get_support() result_nb_out = pd.DataFrame(result_nb) result_nb_out.to_csv('nb_interpretavel.csv') #Avaliacao do SVC classificador_svm = LinearSVC() # Sequential Forward Selection(sfs) sfs = SequentialFeatureSelector(classificador_svm, n_features_to_select=x_train.shape[1] * 0.15) sfs.fit(x_train, y_train) result_svc = sfs.get_support() result_svc_out = pd.DataFrame(result_svc) result_svc_out.to_csv('svc_interpretavel.csv')
def test_n_features_to_select_stopping_criterion(direction): """Check the behaviour stopping criterion for feature selection depending on the values of `n_features_to_select` and `tol`. When `direction` is `'forward'`, select a new features at random among those not currently selected in selector.support_, build a new version of the data that includes all the features in selector.support_ + this newly selected feature. And check that the cross-validation score of the model trained on this new dataset variant is lower than the model with the selected forward selected features or at least does not improve by more than the tol margin. When `direction` is `'backward'`, instead of adding a new feature to selector.support_, try to remove one of those selected features at random And check that the cross-validation score is either decreasing or not improving by more than the tol margin. """ X, y = make_regression(n_features=50, n_informative=10, random_state=0) tol = 1e-3 sfs = SequentialFeatureSelector( LinearRegression(), n_features_to_select="auto", tol=tol, direction=direction, cv=2, ) sfs.fit(X, y) selected_X = sfs.transform(X) rng = np.random.RandomState(0) added_candidates = list( set(range(X.shape[1])) - set(sfs.get_support(indices=True))) added_X = np.hstack([ selected_X, (X[:, rng.choice(added_candidates)])[:, np.newaxis], ]) removed_candidate = rng.choice(list(range(sfs.n_features_to_select_))) removed_X = np.delete(selected_X, removed_candidate, axis=1) plain_cv_score = cross_val_score(LinearRegression(), X, y, cv=2).mean() sfs_cv_score = cross_val_score(LinearRegression(), selected_X, y, cv=2).mean() added_cv_score = cross_val_score(LinearRegression(), added_X, y, cv=2).mean() removed_cv_score = cross_val_score(LinearRegression(), removed_X, y, cv=2).mean() assert sfs_cv_score >= plain_cv_score if direction == "forward": assert (sfs_cv_score - added_cv_score) <= tol assert (sfs_cv_score - removed_cv_score) >= tol else: assert (added_cv_score - sfs_cv_score) <= tol assert (removed_cv_score - sfs_cv_score) <= tol