def test_randomized_lasso(): """Check randomized lasso""" scaling = 0.3 selection_threshold = 0.5 # or with 1 alpha clf = RandomizedLasso(verbose=False, alpha=1, random_state=42, scaling=scaling, selection_threshold=selection_threshold) feature_scores = clf.fit(X, y).scores_ assert_equal(np.argsort(F)[-3:], np.argsort(feature_scores)[-3:]) # or with many alphas clf = RandomizedLasso(verbose=False, alpha=[1, 0.8], random_state=42, scaling=scaling, selection_threshold=selection_threshold) feature_scores = clf.fit(X, y).scores_ assert_equal(clf.all_scores_.shape, (X.shape[1], 2)) assert_equal(np.argsort(F)[-3:], np.argsort(feature_scores)[-3:]) X_r = clf.transform(X) X_full = clf.inverse_transform(X_r) assert_equal(X_r.shape[1], np.sum(feature_scores > selection_threshold)) assert_equal(X_full.shape, X.shape) clf = RandomizedLasso(verbose=False, alpha='aic', random_state=42, scaling=scaling) feature_scores = clf.fit(X, y).scores_ assert_equal(feature_scores, X.shape[1] * [1.]) clf = RandomizedLasso(verbose=False, scaling=-0.1) assert_raises(ValueError, clf.fit, X, y) clf = RandomizedLasso(verbose=False, scaling=1.1) assert_raises(ValueError, clf.fit, X, y)
def set_selection_method(config): """ Given the configuration settings, this function instantiates the configured feature selection method initialized with the preset parameters. TODO: implement the same method using reflection (load the class dinamically at runtime) @param config: the configuration file object loaded using yaml.load() @return: an object that implements the TransformerMixin class (with fit(), fit_transform() and transform() methods). """ transformer = None selection_cfg = config.get("feature_selection", None) if selection_cfg: method_name = selection_cfg.get("method", None) # checks for RandomizedLasso if method_name == "RandomizedLasso": p = selection_cfg.get("parameters", None) if p: transformer = \ RandomizedLasso(alpha=p.get("alpha", "aic"), scaling=p.get("scaling", .5), sample_fraction=p.get('sample_fraction', .75), n_resampling=p.get('n_resampling', 200), selection_threshold=p.get('selection_threshold', .25), fit_intercept=p.get('fit_intercept', True), # TODO: set verbosity according to global level verbose=True, normalize=p.get('normalize', True), max_iter=p.get('max_iter', 500), n_jobs=p.get('n_jobs', 1)) else: transformer = RandomizedLasso() # checks for ExtraTreesClassifier elif method_name == "ExtraTreesClassifier": p = selection_cfg.get("parameters", None) if p: transformer = \ ExtraTreesClassifier(n_estimators=p.get('n_estimators', 10), max_depth=p.get('max_depth', None), min_samples_split=p.get('min_samples_split', 1), min_samples_leaf=p.get('min_samples_leaf', 1), min_density=p.get('min_density', 1), max_features=p.get('max_features', 'auto'), bootstrap=p.get('bootstrap', False), compute_importances=p.get('compute_importances', True), n_jobs=p.get('n_jobs', 1), random_state=p.get('random_state', None), # TODO: set verbosity according to global level verbose=True) else: transformer = ExtraTreesClassifier() return transformer
def run_feature_selection(self, data, labels, feature_selector=None, feature_selection_params={}, feature_selection_threshold=.25, plot_filename="./featureselection.pdf" ): p = feature_selection_params transformer = None attributes = {} if feature_selector == "RandomizedLasso": transformer = RandomizedLasso(alpha=p.setdefault("alpha", "aic"), scaling=p.setdefault("scaling", .5), sample_fraction=p.setdefault('sample_fraction', .75), n_resampling=p.setdefault('n_resampling', 200), selection_threshold=feature_selection_threshold, fit_intercept=p.setdefault('fit_intercept', True), # TODO: set verbosity according to global level verbose=True, normalize=p.setdefault('normalize', True), max_iter=p.setdefault('max_iter', 500), n_jobs=p.setdefault('n_jobs', 1)) elif feature_selector == "ExtraTreesClassifier": transformer = ExtraTreesClassifier(n_estimators=p.get('n_estimators', 10), max_depth=p.get('max_depth', None), min_samples_split=p.get('min_samples_split', 1), min_samples_leaf=p.get('min_samples_leaf', 1), min_density=p.get('min_density', 1), max_features=p.get('max_features', 'auto'), bootstrap=p.get('bootstrap', False), compute_importances=p.get('compute_importances', True), n_jobs=p.get('n_jobs', 1), random_state=p.get('random_state', None), # TODO: set verbosity according to global level verbose=True) elif feature_selector == "GP": #TODO: add here Gaussian Processes transformer = None elif feature_selector == "RFECV_SVC": return self._fs_rfecv(data, labels, plot_filename, sample=p.get("sample", 0.05)) elif feature_selector == "RFE_SVC": return self._fs_rfe(data, labels, plot_filename, n_features=p.get("n_features", 10)) if transformer: log.info("scikit: Running feature selection {}".format(feature_selector)) log.info("scikit: data dimensions before fit_transform(): {}".format(data.shape)) log.info("scikit: labels dimensions before fit_transform(): {}".format(labels.shape)) data = transformer.fit_transform(data, labels) log.info("scikit: Dimensions after fit_transform(): %s,%s" % data.shape) return transformer, data, attributes
def test_randomized_lasso_precompute(): # Check randomized lasso for different values of precompute n_resampling = 20 alpha = 1 random_state = 42 G = np.dot(X.T, X) clf = RandomizedLasso(alpha=alpha, random_state=random_state, precompute=G, n_resampling=n_resampling) feature_scores_1 = clf.fit(X, y).scores_ for precompute in [True, False, None, 'auto']: clf = RandomizedLasso(alpha=alpha, random_state=random_state, precompute=precompute, n_resampling=n_resampling) feature_scores_2 = clf.fit(X, y).scores_ assert_array_equal(feature_scores_1, feature_scores_2)
def test_randomized_lasso_error_memory(): scaling = 0.3 selection_threshold = 0.5 tempdir = 5 clf = RandomizedLasso(verbose=False, alpha=[1, 0.8], random_state=42, scaling=scaling, selection_threshold=selection_threshold, memory=tempdir) assert_raises_regex( ValueError, "'memory' should either be a string or" " a sklearn.externals.joblib.Memory instance", clf.fit, X, y)
def test_randomized_lasso_precompute(): # Check randomized lasso for different values of precompute n_resampling = 20 alpha = 1 random_state = 42 G = np.dot(X.T, X) clf = RandomizedLasso(alpha=alpha, random_state=random_state, precompute=G, n_resampling=n_resampling) feature_scores_1 = clf.fit(X, y).scores_ for precompute in [True, False, None, 'auto']: clf = RandomizedLasso(alpha=alpha, random_state=random_state, precompute=precompute, n_resampling=n_resampling) feature_scores_2 = clf.fit(X, y).scores_ assert_array_equal(feature_scores_1, feature_scores_2)
def test_randomized_lasso(): # Check randomized lasso scaling = 0.3 selection_threshold = 0.5 n_resampling = 20 # or with 1 alpha clf = RandomizedLasso(verbose=False, alpha=1, random_state=42, scaling=scaling, n_resampling=n_resampling, selection_threshold=selection_threshold) feature_scores = clf.fit(X, y).scores_ assert_array_equal(np.argsort(F)[-3:], np.argsort(feature_scores)[-3:]) # or with many alphas clf = RandomizedLasso(verbose=False, alpha=[1, 0.8], random_state=42, scaling=scaling, n_resampling=n_resampling, selection_threshold=selection_threshold) feature_scores = clf.fit(X, y).scores_ assert_equal(clf.all_scores_.shape, (X.shape[1], 2)) assert_array_equal(np.argsort(F)[-3:], np.argsort(feature_scores)[-3:]) # test caching try: tempdir = mkdtemp() clf = RandomizedLasso(verbose=False, alpha=[1, 0.8], random_state=42, scaling=scaling, selection_threshold=selection_threshold, memory=tempdir) feature_scores = clf.fit(X, y).scores_ assert_equal(clf.all_scores_.shape, (X.shape[1], 2)) assert_array_equal(np.argsort(F)[-3:], np.argsort(feature_scores)[-3:]) finally: shutil.rmtree(tempdir) X_r = clf.transform(X) X_full = clf.inverse_transform(X_r) assert_equal(X_r.shape[1], np.sum(feature_scores > selection_threshold)) assert_equal(X_full.shape, X.shape) clf = RandomizedLasso(verbose=False, alpha='aic', random_state=42, scaling=scaling, n_resampling=100) feature_scores = clf.fit(X, y).scores_ assert_allclose(feature_scores, [1., 1., 1., 0.225, 1.], rtol=0.2) clf = RandomizedLasso(verbose=False, scaling=-0.1) assert_raises(ValueError, clf.fit, X, y) clf = RandomizedLasso(verbose=False, scaling=1.1) assert_raises(ValueError, clf.fit, X, y)
def test_randomized_lasso(): # Check randomized lasso scaling = 0.3 selection_threshold = 0.5 # or with 1 alpha clf = RandomizedLasso(verbose=False, alpha=1, random_state=42, scaling=scaling, selection_threshold=selection_threshold) feature_scores = clf.fit(X, y).scores_ assert_array_equal(np.argsort(F)[-3:], np.argsort(feature_scores)[-3:]) # or with many alphas clf = RandomizedLasso(verbose=False, alpha=[1, 0.8], random_state=42, scaling=scaling, selection_threshold=selection_threshold) feature_scores = clf.fit(X, y).scores_ assert_equal(clf.all_scores_.shape, (X.shape[1], 2)) assert_array_equal(np.argsort(F)[-3:], np.argsort(feature_scores)[-3:]) X_r = clf.transform(X) X_full = clf.inverse_transform(X_r) assert_equal(X_r.shape[1], np.sum(feature_scores > selection_threshold)) assert_equal(X_full.shape, X.shape) clf = RandomizedLasso(verbose=False, alpha='aic', random_state=42, scaling=scaling) feature_scores = clf.fit(X, y).scores_ assert_array_equal(feature_scores, X.shape[1] * [1.]) clf = RandomizedLasso(verbose=False, scaling=-0.1) assert_raises(ValueError, clf.fit, X, y) clf = RandomizedLasso(verbose=False, scaling=1.1) assert_raises(ValueError, clf.fit, X, y)
def test_randomized_lasso(): # Check randomized lasso scaling = 0.3 selection_threshold = 0.5 n_resampling = 20 # or with 1 alpha clf = RandomizedLasso(verbose=False, alpha=1, random_state=42, scaling=scaling, n_resampling=n_resampling, selection_threshold=selection_threshold) feature_scores = clf.fit(X, y).scores_ assert_array_equal(np.argsort(F)[-3:], np.argsort(feature_scores)[-3:]) # or with many alphas clf = RandomizedLasso(verbose=False, alpha=[1, 0.8], random_state=42, scaling=scaling, n_resampling=n_resampling, selection_threshold=selection_threshold) feature_scores = clf.fit(X, y).scores_ assert_equal(clf.all_scores_.shape, (X.shape[1], 2)) assert_array_equal(np.argsort(F)[-3:], np.argsort(feature_scores)[-3:]) # test caching try: tempdir = mkdtemp() clf = RandomizedLasso(verbose=False, alpha=[1, 0.8], random_state=42, scaling=scaling, selection_threshold=selection_threshold, memory=tempdir) feature_scores = clf.fit(X, y).scores_ assert_equal(clf.all_scores_.shape, (X.shape[1], 2)) assert_array_equal(np.argsort(F)[-3:], np.argsort(feature_scores)[-3:]) finally: shutil.rmtree(tempdir) X_r = clf.transform(X) X_full = clf.inverse_transform(X_r) assert_equal(X_r.shape[1], np.sum(feature_scores > selection_threshold)) assert_equal(X_full.shape, X.shape) clf = RandomizedLasso(verbose=False, alpha='aic', random_state=42, scaling=scaling, n_resampling=100) feature_scores = clf.fit(X, y).scores_ assert_allclose(feature_scores, [1., 1., 1., 0.225, 1.], rtol=0.2) clf = RandomizedLasso(verbose=False, scaling=-0.1) assert_raises(ValueError, clf.fit, X, y) clf = RandomizedLasso(verbose=False, scaling=1.1) assert_raises(ValueError, clf.fit, X, y)
def get_lasso_feature_scores(x, y, mode=CLASSIFICATION, scaling=0.5, sample_fraction=0.75, n_resampling=200, random_state=None): ''' Calculate features scores using a randomized lasso (regression) or randomized logistic regression (classification). This is also known as stability selection. see http://scikit-learn.org/stable/modules/feature_selection.html for details. Parameters ---------- x : structured array y : 1D nd.array mode : {CLASSIFICATION, REGRESSION} scaling : float, optional scaling parameter, should be between 0 and 1 sample_fraction : float, optional the fraction of samples to used in each randomized dataset n_resmpling : int, optional the number of times the model is trained on a random subset of the data random_state : int, optional if it is an int, it specifies the seed to use, defaults to None. Returns ------- pandas DataFrame sorted in descending order of tuples with uncertainty and feature scores ''' uncs = recfunctions.get_names(x.dtype) x = _prepare_experiments(x) if mode==CLASSIFICATION: lfs = RandomizedLogisticRegression(scaling=scaling, sample_fraction=sample_fraction, n_resampling=n_resampling, random_state=random_state) lfs.fit(x,y) elif mode==REGRESSION: # we use LassoLarsCV to determine alpha see # http://scikit-learn.org/stable/auto_examples/linear_model/plot_sparse_recovery.html lars_cv = LassoLarsCV(cv=6).fit(x, y,) alphas = np.linspace(lars_cv.alphas_[0], .1 * lars_cv.alphas_[0], 6) # fit the randomized lasso lfs = RandomizedLasso(alpha=alphas,scaling=scaling, sample_fraction=sample_fraction, n_resampling=n_resampling, random_state=random_state) lfs.fit(x, y) else: raise ValueError('{} invalid value for mode'.format(mode)) importances = lfs.scores_ importances = zip(uncs, importances) importances = list(importances) importances.sort(key=itemgetter(1), reverse=True) importances = pd.DataFrame(importances) return importances
'PCA':PCA(), 'PLSCanonical':PLSCanonical(), 'PLSRegression':PLSRegression(), 'PLSSVD':PLSSVD(), 'PassiveAggressiveClassifier':PassiveAggressiveClassifier(), 'PassiveAggressiveRegressor':PassiveAggressiveRegressor(), 'Perceptron':Perceptron(), 'ProjectedGradientNMF':ProjectedGradientNMF(), 'QuadraticDiscriminantAnalysis':QuadraticDiscriminantAnalysis(), 'RANSACRegressor':RANSACRegressor(), 'RBFSampler':RBFSampler(), 'RadiusNeighborsClassifier':RadiusNeighborsClassifier(), 'RadiusNeighborsRegressor':RadiusNeighborsRegressor(), 'RandomForestClassifier':RandomForestClassifier(), 'RandomForestRegressor':RandomForestRegressor(), 'RandomizedLasso':RandomizedLasso(), 'RandomizedLogisticRegression':RandomizedLogisticRegression(), 'RandomizedPCA':RandomizedPCA(), 'Ridge':Ridge(), 'RidgeCV':RidgeCV(), 'RidgeClassifier':RidgeClassifier(), 'RidgeClassifierCV':RidgeClassifierCV(), 'RobustScaler':RobustScaler(), 'SGDClassifier':SGDClassifier(), 'SGDRegressor':SGDRegressor(), 'SVC':SVC(), 'SVR':SVR(), 'SelectFdr':SelectFdr(), 'SelectFpr':SelectFpr(), 'SelectFwe':SelectFwe(), 'SelectKBest':SelectKBest(),
def get_lasso_feature_scores(results, classify, scaling=0.5, sample_fraction=0.75, n_resampling=200, random_state=None): ''' Calculate features scores using a randomized lasso (regression) or randomized logistic regression (classification). This is also known as stability selection. see http://scikit-learn.org/stable/modules/feature_selection.html for details. Parameters ---------- results : tuple classify : callable or str a classify function or variable analogous to PRIM scaling : float, optional scaling parameter, should be between 0 and 1 sample_fraction : float, optional the fraction of samples to used in each randomized dataset n_resmpling : int, optional the number of times the model is trained on a random subset of the data random_state : int, optional if it is an int, it specifies the seed to use, defaults to None. Returns ------- list of tuples sorted in descending order of tuples with uncertainty and feature scores ''' experiments, outcomes = results uncs = recfunctions.get_names(experiments.dtype) x = _prepare_experiments(experiments) y, categorical = _prepare_outcomes(outcomes, classify) if categorical: lfs = RandomizedLogisticRegression(scaling=scaling, sample_fraction=sample_fraction, n_resampling=n_resampling, random_state=random_state) lfs.fit(x, y) else: # we use LassoLarsCV to determine alpha see # http://scikit-learn.org/stable/auto_examples/linear_model/plot_sparse_recovery.html lars_cv = LassoLarsCV(cv=6).fit( x, y, ) alphas = np.linspace(lars_cv.alphas_[0], .1 * lars_cv.alphas_[0], 6) # fit the randomized lasso lfs = RandomizedLasso(alpha=alphas, scaling=scaling, sample_fraction=sample_fraction, n_resampling=n_resampling, random_state=random_state) lfs.fit(x, y) importances = lfs.scores_ importances = zip(uncs, importances) importances = list(importances) importances.sort(key=itemgetter(1), reverse=True) return importances