from pyxvis.io.data import load_features from pyxvis.io.plots import show_clf_results from pyxvis.learning.classifiers import clf_model, define_classifier from pyxvis.learning.classifiers import train_classifier, test_classifier (X, d, Xt, dt) = load_features('../data/F2/F2') # load training and testing data ss_cl = ['nn(10,)', 'nn(12,6)'] for cl_name in ss_cl: (name, params) = clf_model(cl_name) # function name and parameters clf = define_classifier([name, params]) # classifier definition clf = train_classifier(clf, X, d) # classifier training d0 = test_classifier(clf, X) # classification of training ds = test_classifier(clf, Xt) # classification of testing show_clf_results(clf, X, d, Xt, dt, d0, ds, cl_name) # display results and decision lines
from pyxvis.learning.evaluation import cross_validation # List of classifiers ss_cl = [ 'dmin', 'lda', 'qda', 'maha', 'knn3', 'knn5', 'knn7', 'knn11', 'knn15', 'bayes-naive', 'bayes-kde', 'adaboost', 'lr', 'rf', 'tree', 'svm-lin', 'svm-rbf(0.1,1)', 'svm-rbf(0.1,0.5)', 'svm-rbf(0.5,1)', 'svm-pol(0.05,0.1,2)', 'svm-pol(0.05,0.5,2)', 'svm-pol(0.05,0.5,3)', 'svm-sig(0.1,1)', 'svm-sig(0.1,0.5)', 'svm-sig(0.5,1)', 'nn(10,)', 'nn(20,)', 'nn(12,6)', 'nn(20,10,4)' ] (X, d) = load_features('../data/G3/G3', full=1) # load training and testing data n = len(ss_cl) folds = 10 acc = np.zeros((n, )) for k in range(n): (name, params) = clf_model(ss_cl[k]) # function name and parameters acc[k] = cross_validation([name, params], X, d, folds=folds) print(f'{k:3d}' + ') ' + f'{ss_cl[k]:20s}' + ': ' + f'CV-Accuracy = {acc[k]:.4f}') ks = np.argmax(acc) print('-----------------------------------------------') print('Best Classifier:') print(f'{ks:3d}' + ') ' + f'{ss_cl[ks]:20s}' + ': ' + f'CV-Accuracy = {acc[ks]:.4f}') print('-----------------------------------------------')
def best_features_classifier(ss_fs,ff,ss_cl,X,d,Xt,dt,stit,folds=5): ncl = len(ss_cl) # number of classifiers nfs = len(ss_fs) # number of feature selectors nfk = len(ff) # number of feature selectors accbest = 0 # best achieved accuracy acctbest = 0 # best achieved accuracy yacc = np.zeros((1000,)) yacct = np.zeros((1000,)) nt = np.zeros((1000,)) clfs = [None] * 1000 fses = [None] * 1000 t = 1 for k in range(nfk): nfik = ff[k] for i in range(nfs): fs = ss_fs[i] print('analyzing feature felector: sfs+'+fs+' with '+str(nfik)+' features...') (fsname,fspar) = fse_model(fs) ss = fsel([fsname,fspar],X,d,nfik,cv = folds, show = 1) Xf = X[:,ss] Xft = Xt[:,ss] for j in range(ncl): cl = ss_cl[j] # print('analyzing classifier '+cl+'...') (clname,clpar) = clf_model(cl) acc = cross_validation([clname,clpar],Xf,d,folds) _,acct,_ = hold_out([clname,clpar],Xf,d,Xft,dt) stn = f'{nfik:2d}' stt = 'feat='+stn+', fsel='+fs+', clf='+cl stb = '' sti = '(iter = '+str(t)+')' if acc>accbest: accbest = acc stb = sti + ' *** new train max found ***' yacc[t] = acc yacct[t] = acct nt[t] = nfik clfs[t] = cl fses[t] = fs if acct > acctbest: stb = sti + ' *** new train/test max found ***' tbest = t acctbest = acct accbest0 = acc fsnbest = fsname fspbest = fspar fsbest = fs clbest = cl clnbest = clname clpbest = clpar nbest = nfik ssbest = ss t = t + 1 print(f'{stt:38s} > acc = {acc:.4f}/{acct:.4f} <best acc = {accbest:.4f}/{acctbest:.4f}> (Train/Test) '+stb) print('---------------------------------------------------------------------------') print(' Best iteration: '+str(tbest)+' (maximum of testing accuracy)') print(' Feature Selector: ' +fsbest+' with '+str(nbest)+' features') print(' : ('+fsnbest+', '+fspbest+')') print(' Classifier: ' +clbest) print(' : ('+clnbest+', '+clpbest+') CrossVal with '+str(folds)+' folds') print(f' Training-Acc: {accbest0:.4f} ') print(f' Testing-Acc: {acctbest:.4f} ') y1 = yacc[0:t] y2 = yacct[0:t] x = range(t) # fig = plt.figure() fig = plt.figure(figsize=(t+2,5)) ax = fig.add_subplot(111) plt.plot(x[1:t], y1[1:t], marker='o', markerfacecolor='blue', markersize=6, label = 'Training') plt.plot(x[1:t], y2[1:t], marker='o', markerfacecolor='red', markersize=6, label = 'Testing') plt.xlabel('iterations') plt.ylabel('accuracy') plt.title(stit) yf = 0.405 dy = 0.05 dd = 0.02 fts = 10 ax.text(dd, yf, 'features:', fontsize=fts) ax.text(dd, yf-dy, 'f-selector:', fontsize=fts) ax.text(dd, yf-dy*2, 'classifier:', fontsize=fts) ax.text(dd, yf-dy*3, 'acc-train:', fontsize=fts) ax.text(dd, yf-dy*4, 'acc-test:', fontsize=fts) for i in range(1,t): col = 'black' if i==tbest: col = 'green' di = i+dd ax.text(di, yf, str(int(nt[i])), fontsize=fts,color=col) ax.text(di, yf-dy, fses[i], fontsize=fts,color=col) ax.text(di, yf-dy*2, clfs[i], fontsize=fts,color=col) accs = f'{y1[i]:.4f}' accts = f'{y2[i]:.4f}' ax.text(di, yf-dy*3, accs, fontsize=fts,color=col) ax.text(di, yf-dy*4, accts, fontsize=fts,color=col) ax.text(tbest-0.5, yf+0.2, 'best test accuracy', fontsize=fts,color='green') ax.annotate(' ',xy=(tbest, acctbest-0.05), xytext=(tbest, yf+0.2), arrowprops=dict(facecolor='green', shrink=0.05)) ax.plot(tbest,acctbest, 'o') plt.xticks(range(t+1)) plt.grid(True) plt.legend() plt.xlim(0, t) plt.ylim(0, 1.1) plt.show() return clbest, ssbest