def inc(fn,title,do_plot=0,eyl=1): l = AttrDict(load(fn+"pred.npz")) Ym, Ypred, expa = l.Ym, l.Ypred, l.expa print title print 'RMSEP',RMSEP(Ym, Ypred), 'BIAS',BIAS(Ym, Ypred), 'SE',SE(Ym, Ypred) #print len(Ym),len(Ypred),len(expa) if not do_plot: return persons = Persons(expa) ax = fig_pred.subplot(do_plot) Ymm = [0,max(Ym)] ax.plot(Ymm,Ymm,'g-') persons.plot( ax, Ym, Ypred ) ax.set_title(title) ax.set_xlabel('IS, measured, mg/L') if eyl: ax.set_ylabel('IS, predicted, mg/L') ax = fig_ba.subplot(do_plot) mean_mp = (Ym+Ypred)/2. diff_mp = Ym-Ypred mm_mean_mp = [min(mean_mp),max(mean_mp)] md,s2 = mean(diff_mp), 2*diff_mp.std() #print mm_mean_mp,md,s2 #ax.clabel(contour,'AAA')#ax.set_xticklabels(['EE'],minor=True) def hline(l,txt): ax.axhline(y=l,ls=':',c='g') if txt: ax.annotate(txt,(mm_mean_mp[0],l),color='g') hline(md-s2,'Mean-2*SD') hline(md,'') hline(md+s2,'Mean+2*SD') persons.plot( ax, mean_mp, diff_mp ) ax.set_title(title+', Bland-Altman') ax.set_xlabel('IS, (measured+predicted)/2, mg/L') if eyl: ax.set_ylabel('IS, (measured-predicted), mg/L')
print "predict..." Y_pred = pls.predict(X4test.copy()) dis4test = Y4test[:,0] dis_pred = Y_pred[:,0] dis_max = max(disa) #dis_pred = where(dis_pred<dis_max+1,where(dis_pred<-1,-1,dis_pred),dis_max+1) persons = Persons(expa4test) #print ia4test, ia4fit, logical_not(a4fit & good_std) #print expa4test.shape, dis4test.shape, dis_pred.shape, Y.shape, Y4test.shape, Y_pred.shape plt.plot([0,dis_max],[0,dis_max],'g-') persons.plot(plt,dis4test,dis_pred) plt.savefig(out_pre+"pred.png") plt.cla() print "fit..." pls = PLSRegression(n_components=n_components,algorithm='svd',scale=False) pls.fit(X=X,Y=Y) print "save..." """ print pls.x_loadings_.shape, pls.x_weights_.shape, pls.x_rotations_.shape, \ pls.y_loadings_.shape """ savez(out_pre+"loadings.npz", x_loadings = pls.x_loadings_,
n_components = 14 from numpy import load, array, where from numpy.linalg import norm from linre_tools import AttrDict from persons import Persons import matplotlib.pyplot as plt l = AttrDict(load('linre_big'+'2'+'.npz')) mds = AttrDict(load("out23/pred.npz")) X, Y = mds.X, mds.Y Ypred4n = mds.Ypred4n_components Ym, Ypred = Y[:,0], Ypred4n[:,n_components-1] eee = l.ema==l.exa Xeee = X[:,eee] #efm = Xeee.mean(axis=1) #efm = [norm(s) for s in Xeee] #efm = [len(where(s)[0]) for s in X_err] #print Ym.shape, efm.shape persons = Persons(mds.expa) persons.plot( plt, efm, Ypred - Ym) plt.show()
var_count = X.shape[1] ma = empty((var_count,)) disa_pred4var = empty_like(X) for varn in range(var_count): Xc = X[:,varn] loo = KFold( n=len(disa), k=group_count, indices=False ) for fit, test in loo: slope, intercept, r_value, p_value, stderr = linregress(Xc[fit],disa[fit]) disa_pred4var[test,varn] = Xc[test] * slope + intercept ma[varn] = RMSEP(disa,disa_pred4var[:,varn]) ia = argmin(ma) print ma[ia] l = AttrDict(load('linre_big2.npz')) print l.exa[ia], l.ema[ia] persons = Persons(expa) Ymm = [0,max(disa)] plt.plot(Ymm,Ymm,'g-') persons.plot(plt,disa,disa_pred4var[:,ia]) plt.title('Univariate, K-Fold, '+str(len(disa))+' samples') plt.xlabel('IS, measured, mg/L') plt.ylabel('IS, predicted, mg/L') plt.savefig("out26/pred.png") Ym, Ypred = disa, disa_pred4var[:,ia] print 'RMSEP', RMSEP(Ym, Ypred) print 'BIAS', BIAS(Ym, Ypred) print 'SE', SE(Ym, Ypred) savez('out26/pred.npz',Ym=Ym,Ypred=Ypred,expa=expa)
X_orig = flum.T X_err, X_orig = find_peaks(X_orig,exa) ## exclude outliers PC = PCA(n_components=2).fit_transform(X_orig.copy()) #mean inside PC1 = PC[:,0] good_std = PC1 < PC1.std() a4fit = arange(len(X_orig)) >= samples_in_testing_set ia4fit, = where( a4fit & good_std ) X4fit = X_orig[ia4fit,:] disa4fit = disa[ia4fit] pca = PCA(n_components=n_components) PC = pca.fit_transform(X4fit.copy()) dis_mean = disa4fit.mean() #print PC.shape,(disa4fit-dis_mean).shape (a,residues,rank,s) = lstsq(PC,disa4fit-dis_mean) PC = pca.transform(X_orig.copy()) mdis = PC.dot(a[:,None])[:,0] + dis_mean from persons import Persons persons = Persons(expa) import matplotlib.pyplot as plt plt.figure() dis_max = max(disa) plt.plot([0,dis_max],[0,dis_max],'g-') persons.plot(plt,disa,where(mdis<dis_max+1,where(mdis<-1,-1,mdis),dis_max+1)) plt.show()