def test_cvbestsearchrefit(self): X, y = datasets.make_classification(n_samples=12, n_features=10, n_informative=2) n_folds_nested = 2 #random_state = 0 C_values = [.1, 0.5, 1, 2, 5] kernels = ["linear", "rbf"] key_y_pred = 'y' + conf.SEP + conf.PREDICTION # With EPAC methods = Methods(*[SVC(C=C, kernel=kernel) for C in C_values for kernel in kernels]) wf = CVBestSearchRefit(methods, n_folds=n_folds_nested) wf.run(X=X, y=y) r_epac = wf.reduce().values()[0] # - Without EPAC r_sklearn = dict() clf = SVC(kernel="linear") parameters = {'C': C_values, 'kernel': kernels} cv_nested = StratifiedKFold(y=y, n_folds=n_folds_nested) gscv = grid_search.GridSearchCV(clf, parameters, cv=cv_nested) gscv.fit(X, y) r_sklearn[key_y_pred] = gscv.predict(X) r_sklearn[conf.BEST_PARAMS] = gscv.best_params_ # - Comparisons comp = np.all(r_epac[key_y_pred] == r_sklearn[key_y_pred]) self.assertTrue(comp, u'Diff CVBestSearchRefit: prediction') for key_param in r_epac[conf.BEST_PARAMS][0]: if key_param in r_sklearn[conf.BEST_PARAMS]: comp = r_sklearn[conf.BEST_PARAMS][key_param] == \ r_epac[conf.BEST_PARAMS][0][key_param] self.assertTrue(comp, \ u'Diff CVBestSearchRefit: best parameters')
def test_mem(): X, y = datasets.make_classification(n_samples=2000, n_features=10000, n_informative=2, random_state=1) wf = CVBestSearchRefit( Methods(*[SVC(kernel="linear"), SVC(kernel="rbf")]), n_folds=10) wf.run(X=X, y=y) # Top-down process: computing recognition rates, etc. print wf.reduce() # Bottom-up process: computing p-values, etc.
def test_peristence_perm_cv_parmethods_pipe_vs_sklearn(self): key_y_pred = 'y' + conf.SEP + conf.PREDICTION X, y = datasets.make_classification(n_samples=12, n_features=10, n_informative=2) n_folds_nested = 2 #random_state = 0 C_values = [.1, 0.5, 1, 2, 5] kernels = ["linear", "rbf"] # With EPAC methods = Methods(*[SVC(C=C, kernel=kernel) for C in C_values for kernel in kernels]) wf = CVBestSearchRefit(methods, n_folds=n_folds_nested) # Save workflow # ------------- import tempfile #store = StoreFs("/tmp/toto", clear=True) store = StoreFs(tempfile.mktemp()) wf.save_tree(store=store) wf = store.load() wf.run(X=X, y=y) ## Save results wf.save_tree(store=store) wf = store.load() r_epac = wf.reduce().values()[0] # - Without EPAC r_sklearn = dict() clf = SVC(kernel="linear") parameters = {'C': C_values, 'kernel': kernels} cv_nested = StratifiedKFold(y=y, n_folds=n_folds_nested) gscv = grid_search.GridSearchCV(clf, parameters, cv=cv_nested) gscv.fit(X, y) r_sklearn[key_y_pred] = gscv.predict(X) r_sklearn[conf.BEST_PARAMS] = gscv.best_params_ r_sklearn[conf.BEST_PARAMS]['name'] = 'SVC' # - Comparisons comp = np.all(r_epac[key_y_pred] == r_sklearn[key_y_pred]) self.assertTrue(comp, u'Diff CVBestSearchRefit: prediction') comp = np.all([r_epac[conf.BEST_PARAMS][0][p] == r_sklearn[conf.BEST_PARAMS][p] for p in r_sklearn[conf.BEST_PARAMS]]) self.assertTrue(comp, u'Diff CVBestSearchRefit: best parameters')
def test_cvbestsearchrefit_select_k_best(self): list_C_value = range(2, 10, 1) # print repr(list_C_value) for C_value in list_C_value: # C_value = 2 # print C_value X, y = datasets.make_classification(n_samples=100, n_features=500, n_informative=5) n_folds_nested = 2 #random_state = 0 k_values = [2, 3, 4, 5, 6] key_y_pred = 'y' + conf.SEP + conf.PREDICTION # With EPAC methods = Methods(*[Pipe(SelectKBest(k=k), SVC(C=C_value, kernel="linear")) for k in k_values]) wf = CVBestSearchRefit(methods, n_folds=n_folds_nested) wf.run(X=X, y=y) r_epac = wf.reduce().values()[0] # - Without EPAC from sklearn.pipeline import Pipeline r_sklearn = dict() clf = Pipeline([('anova', SelectKBest(k=3)), ('svm', SVC(C=C_value, kernel="linear"))]) parameters = {'anova__k': k_values} cv_nested = StratifiedKFold(y=y, n_folds=n_folds_nested) gscv = grid_search.GridSearchCV(clf, parameters, cv=cv_nested) gscv.fit(X, y) r_sklearn[key_y_pred] = gscv.predict(X) r_sklearn[conf.BEST_PARAMS] = gscv.best_params_ r_sklearn[conf.BEST_PARAMS]['k'] = \ r_sklearn[conf.BEST_PARAMS]['anova__k'] # - Comparisons comp = np.all(r_epac[key_y_pred] == r_sklearn[key_y_pred]) self.assertTrue(comp, u'Diff CVBestSearchRefit: prediction') for key_param in r_epac[conf.BEST_PARAMS][0]: if key_param in r_sklearn[conf.BEST_PARAMS]: comp = r_sklearn[conf.BEST_PARAMS][key_param] == \ r_epac[conf.BEST_PARAMS][0][key_param] self.assertTrue(comp, \ u'Diff CVBestSearchRefit: best parameters')
cv_results = cv.reduce() print cv_results #[{'key': CVBestSearchRefit, 'y/test/score_f1': [ 0.82352941 0.7 ], 'y/test/recall_pvalues': [ 0.01086887 0.06790736], 'y/test/score_precision': [ 0.77777778 0.77777778], 'y/test/recall_mean_pvalue': 0.0191572904587, 'y/test/score_recall': [ 0.875 0.63636364], 'y/test/score_accuracy': 0.777777777778, 'y/test/score_recall_mean': 0.755681818182}]) # #Parmis les 27 11 ont fait la transition et 16 ne l'on pas faite #- Sensibilité (Taux de detection de les transitions) #63.63 % soit 7 / 11 (p = 0.067) # #- Spécificité (Taux de detection de ceux qui n'ont pas transité ou 1 - Faux positifs) #87.5 % soit 14 / 16 (p = 0.01) # #Nous avons un taux de bonne classification moyen de 77 % # svms_cv.run(X=X, y=y) #{'best_params': [{'C': 0.05, 'name': 'LinearSVC'}], # 'y/pred': array([0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, # 0, 0, 0, 0]), # 'y/true': array([0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, # 0, 0, 0, 0])} svm = SVM(dual=False, class_weight='auto', penalty="l1", C=.05) svm.fit(X, y) svm.coef_ len(svm.coef_.squeeze()) # 27 coef = svm.coef_.squeeze() print pd.DataFrame(dict(var=Xd.columns[coef != 0], coef=coef[coef != 0])) # coef var #0 -0.084406 @4.3
# LDA SVM Classifier (Estimator) from epac import CV, Methods cv = CV(Methods(LDA(), SVM())) cv.run(X=X, y=y) print(cv.reduce()) # Model selection using CV # ------------------------ # CVBestSearchRefit # Methods (Splitter) # / \ # SVM(C=1) SVM(C=10) Classifier (Estimator) from epac import Pipe, CVBestSearchRefit, Methods # CV + Grid search of a simple classifier wf = CVBestSearchRefit(Methods(SVM(C=1), SVM(C=10))) wf.run(X=X, y=y) print(wf.reduce()) # Feature selection combined with SVM and LDA # CVBestSearchRefit # Methods (Splitter) # / \ # KBest(1) KBest(5) SelectKBest (Estimator) # | # Methods (Splitter) # / \ # LDA() SVM() ... Classifiers (Estimator) pipelines = Methods( *[Pipe(SelectKBest(k=k), Methods(LDA(), SVM())) for k in [1, 5]]) print([n for n in pipelines.walk_leaves()]) best_cv = CVBestSearchRefit(pipelines)
from epac import CV, Methods cv = CV(Methods(LDA(), SVM())) cv.run(X=X, y=y) print cv.reduce() # Model selection using CV # ------------------------ # CVBestSearchRefit # Methods (Splitter) # / \ # SVM(C=1) SVM(C=10) Classifier (Estimator) from epac import Pipe, CVBestSearchRefit, Methods # CV + Grid search of a simple classifier wf = CVBestSearchRefit(Methods(SVM(C=1), SVM(C=10))) wf.run(X=X, y=y) print wf.reduce() # Feature selection combined with SVM and LDA # CVBestSearchRefit # Methods (Splitter) # / \ # KBest(1) KBest(5) SelectKBest (Estimator) # | # Methods (Splitter) # / \ # LDA() SVM() ... Classifiers (Estimator) pipelines = Methods(*[Pipe(SelectKBest(k=k), Methods(LDA(), SVM())) for k in [1, 5]]) print [n for n in pipelines.walk_leaves()] best_cv = CVBestSearchRefit(pipelines) best_cv.run(X=X, y=y)
print pd.DataFrame(d).to_string() ############################################################################## # Automatic model selection: "CVBestSearchRefit" from epac import CVBestSearchRefit, Methods, CV svms_auto = CVBestSearchRefit(svms) cv = CV(svms_auto, n_folds=n_folds) cv.run(X=X, y=y) # res_cv_svms_auto = cv.reduce() print res_cv_svms_auto print res_cv_svms_auto["CVBestSearchRefit"]['y/test/score_recall'] # Re-fit on all data. Warning: biased !!! svms_auto.run(X=X, y=y) print svms_auto.best_params print svms_auto.refited.estimator.coef_ ############################################################################## # Put everything together # Pipeline, "Pipe": SelectKBest + StandardScaler + SVM l1 vs l2 from epac import range_log2 from epac import CVBestSearchRefit, Pipe, Methods, CV k_values = range_log2(X.shape[1], add_n=True) C_values = [.1, 1, 10, 100] anova_svms = Methods(*[Pipe(SelectKBest(k=k), preprocessing.StandardScaler(), Methods(*[SVM(C=C, penalty=penalty, class_weight='auto', dual=False) for C in C_values for penalty in ['l1', 'l2']])) for k in k_values])