def inc(fn,title,do_plot=0,eyl=1):
l = AttrDict(load(fn+"pred.npz"))
Ym, Ypred, expa = l.Ym, l.Ypred, l.expa
print title
print 'RMSEP',RMSEP(Ym, Ypred), 'BIAS',BIAS(Ym, Ypred), 'SE',SE(Ym, Ypred)
#print len(Ym),len(Ypred),len(expa)
if not do_plot: return
persons = Persons(expa)
ax = fig_pred.subplot(do_plot)
Ymm = [0,max(Ym)]
ax.plot(Ymm,Ymm,'g-')
persons.plot( ax, Ym, Ypred )
ax.set_title(title)
ax.set_xlabel('IS, measured, mg/L')
if eyl: ax.set_ylabel('IS, predicted, mg/L')
ax = fig_ba.subplot(do_plot)
mean_mp = (Ym+Ypred)/2.
diff_mp = Ym-Ypred
mm_mean_mp = [min(mean_mp),max(mean_mp)]
md,s2 = mean(diff_mp), 2*diff_mp.std()
#print mm_mean_mp,md,s2 #ax.clabel(contour,'AAA')#ax.set_xticklabels(['EE'],minor=True)
def hline(l,txt):
ax.axhline(y=l,ls=':',c='g')
if txt: ax.annotate(txt,(mm_mean_mp[0],l),color='g')
hline(md-s2,'Mean-2*SD')
hline(md,'')
hline(md+s2,'Mean+2*SD')
persons.plot( ax, mean_mp, diff_mp )
ax.set_title(title+', Bland-Altman')
ax.set_xlabel('IS, (measured+predicted)/2, mg/L')
if eyl: ax.set_ylabel('IS, (measured-predicted), mg/L')
print "predict..."
Y_pred = pls.predict(X4test.copy())
dis4test = Y4test[:,0]
dis_pred = Y_pred[:,0]
dis_max = max(disa)
#dis_pred = where(dis_pred<dis_max+1,where(dis_pred<-1,-1,dis_pred),dis_max+1)
persons = Persons(expa4test)
#print ia4test, ia4fit, logical_not(a4fit & good_std)
#print expa4test.shape, dis4test.shape, dis_pred.shape, Y.shape, Y4test.shape, Y_pred.shape
plt.plot([0,dis_max],[0,dis_max],'g-')
persons.plot(plt,dis4test,dis_pred)
plt.savefig(out_pre+"pred.png")
plt.cla()
print "fit..."
pls = PLSRegression(n_components=n_components,algorithm='svd',scale=False)
pls.fit(X=X,Y=Y)
print "save..."
"""
print pls.x_loadings_.shape, pls.x_weights_.shape, pls.x_rotations_.shape, \
pls.y_loadings_.shape
"""
savez(out_pre+"loadings.npz",
x_loadings = pls.x_loadings_,
n_components = 14
from numpy import load, array, where
from numpy.linalg import norm
from linre_tools import AttrDict
from persons import Persons
import matplotlib.pyplot as plt
l = AttrDict(load('linre_big'+'2'+'.npz'))
mds = AttrDict(load("out23/pred.npz"))
X, Y = mds.X, mds.Y
Ypred4n = mds.Ypred4n_components
Ym, Ypred = Y[:,0], Ypred4n[:,n_components-1]
eee = l.ema==l.exa
Xeee = X[:,eee]
#efm = Xeee.mean(axis=1)
#efm = [norm(s) for s in Xeee]
#efm = [len(where(s)[0]) for s in X_err]
#print Ym.shape, efm.shape
persons = Persons(mds.expa)
persons.plot( plt, efm, Ypred - Ym)
plt.show()
var_count = X.shape[1]
ma = empty((var_count,))
disa_pred4var = empty_like(X)
for varn in range(var_count):
Xc = X[:,varn]
loo = KFold( n=len(disa), k=group_count, indices=False )
for fit, test in loo:
slope, intercept, r_value, p_value, stderr = linregress(Xc[fit],disa[fit])
disa_pred4var[test,varn] = Xc[test] * slope + intercept
ma[varn] = RMSEP(disa,disa_pred4var[:,varn])
ia = argmin(ma)
print ma[ia]
l = AttrDict(load('linre_big2.npz'))
print l.exa[ia], l.ema[ia]
persons = Persons(expa)
Ymm = [0,max(disa)]
plt.plot(Ymm,Ymm,'g-')
persons.plot(plt,disa,disa_pred4var[:,ia])
plt.title('Univariate, K-Fold, '+str(len(disa))+' samples')
plt.xlabel('IS, measured, mg/L')
plt.ylabel('IS, predicted, mg/L')
plt.savefig("out26/pred.png")
Ym, Ypred = disa, disa_pred4var[:,ia]
print 'RMSEP', RMSEP(Ym, Ypred)
print 'BIAS', BIAS(Ym, Ypred)
print 'SE', SE(Ym, Ypred)
savez('out26/pred.npz',Ym=Ym,Ypred=Ypred,expa=expa)
X_orig = flum.T X_err, X_orig = find_peaks(X_orig,exa) ## exclude outliers PC = PCA(n_components=2).fit_transform(X_orig.copy()) #mean inside PC1 = PC[:,0] good_std = PC1 < PC1.std() a4fit = arange(len(X_orig)) >= samples_in_testing_set ia4fit, = where( a4fit & good_std ) X4fit = X_orig[ia4fit,:] disa4fit = disa[ia4fit] pca = PCA(n_components=n_components) PC = pca.fit_transform(X4fit.copy()) dis_mean = disa4fit.mean() #print PC.shape,(disa4fit-dis_mean).shape (a,residues,rank,s) = lstsq(PC,disa4fit-dis_mean) PC = pca.transform(X_orig.copy()) mdis = PC.dot(a[:,None])[:,0] + dis_mean from persons import Persons persons = Persons(expa) import matplotlib.pyplot as plt plt.figure() dis_max = max(disa) plt.plot([0,dis_max],[0,dis_max],'g-') persons.plot(plt,disa,where(mdis<dis_max+1,where(mdis<-1,-1,mdis),dis_max+1)) plt.show()
