def buildwPLS(Dm,R,A,*,num_si_u=0,R_ik=False):
"""
PLS-EIOT BUILD Routines by Salvador Garcia-Munoz ([email protected], [email protected])
Parameters
----------
Dm : TYPE, Numpy Array.
DESCRIPTION. Mixture spectra [o x lambda].
R : TYPE, Numpy Array.
DESCRIPTION. Compositional matrix [o x n] with mass/mole/vol fractions.
A : TYPE, Integer.
DESCRIPTION. Number of latent variables to use in PLS model.
num_si_u : TYPE, optional.
DESCRIPTION. Number of un-supervised non-chemical interferences, the default is 0.
R_ik : TYPE, Numpy Array.
DESCRIPTION. Matrix with supervised non-chemical interferences The default is False.
Returns
-------
eiot_pred_obj a Dictionary with predicions and diagnostics
"""
rk=R_ik
Ck=R
num_sI=num_si_u
if isinstance(rk,bool):
print('Building a PLS based EIOT Unsupervised object')
pls_obj=phi.pls(Ck,Dm,A)
Dm_hat=phi.pls_pred(Ck,pls_obj)
else:
print('Building a PLS based EIOT Supervised object')
aux=np.hstack((Ck,rk))
pls_obj=phi.pls(aux,Dm,A)
Dm_hat=phi.pls_pred(aux,pls_obj)
pls_obj=phi.conv_pls_2_eiot(pls_obj,r_length=Ck.shape[1])
Dm_hat=Dm_hat['Yhat']
if num_sI>0:
E_ch = Dm-Dm_hat
[U,S,Vh] = np.linalg.svd(E_ch)
V = Vh.T
S_I = V[:,0:num_sI]
S_short = S[0:num_sI]
r_I = U[:,0:num_sI] * np.tile(S_short,(U.shape[0],1))
SR = E_ch- r_I @ S_I.T
SSR = np.sum(SR**2,1,keepdims=1)
lambdas = S[num_sI]
else:
S_I = np.nan
SR = Dm-Dm_hat
E_ch = SR
[U,S,Vh] = np.linalg.svd(E_ch)
V = Vh.T
lambdas = np.diag(S)
lambdas = np.diag(lambdas)
SSR = np.sum(SR**2,axis=1)
r_I = np.nan
if not(isinstance(S_I,float)):
S_I=S_I.T
eiot_obj = pls_obj
eiot_obj['wPLS'] = True
eiot_obj['S_I'] = S_I
eiot_obj['E_ch'] = E_ch
eiot_obj['r_I'] = r_I
eiot_obj['SR'] = SR
eiot_obj['SSR'] = SSR
eiot_obj['num_sI'] = num_sI
eiot_obj['num_e_sI'] = 0
eiot_obj['abs_max_exc_ri'] = np.nan
eiot_obj['S_E_CONF_INT'] = np.nan
#Convert Numpy objects to lists and dict for PYOMO
if not(isinstance(S_I,float)):
pyo_S_I = ee.np2D2pyomo(eiot_obj['S_I']) #convert numpy to dictionary
pyo_K = np.arange(1,eiot_obj['S_I'].shape[0]+1) #index for non-chemical interferences
pyo_K = pyo_K.tolist()
else:
pyo_S_I = np.nan
pyo_K = [0]
eiot_obj['pyo_K'] = pyo_K
eiot_obj['pyo_S_I'] = pyo_S_I
eiot_obj['lambdas'] = lambdas
return eiot_obj
xaxis_label='channel',yaxis_label='a.u')
# Create a new Pandas Data Frame with SNV(Raman_Spectra)
Raman_Spectra_snv=phi.snv(Raman_Spectra)
#Use pyphi_plots to plot spectra
pp.plot_spectra(Raman_Spectra_snv,plot_title='Raman with SNV',tab_title='Spectra SNV',
xaxis_label='channel',yaxis_label='a.u')
# Create a new Pandas Data Frame with Pre-processed spectra
# SavGol transform with:
# window_size = 10
# derivative order = 1
# polynomial order = 2
Raman_Spectra_snv_savgol,M=phi.savgol(10,1,2,Raman_Spectra_snv)
#Use pyphi_plots to plot spectra
pp.plot_spectra(Raman_Spectra_snv_savgol,plot_title='Raman with SNV and Savitzky Golay [10,1,2]',tab_title='Spectra SNV+SavGol',
xaxis_label='channel',yaxis_label='a.u')
#Build three models and contrast
pls_raman_calibration=phi.pls(Raman_Spectra,API_Conc,3,mcsX='center',mcsY='center')
pp.predvsobs(pls_raman_calibration,Raman_Spectra,API_Conc)
pls_raman_calibration=phi.pls(Raman_Spectra_snv,API_Conc,3,mcsX='center',mcsY='center',cross_val=10)
pp.predvsobs(pls_raman_calibration,Raman_Spectra_snv,API_Conc)
pls_raman_calibration=phi.pls(Raman_Spectra_snv_savgol,API_Conc,1,mcsX='center',mcsY='center',cross_val=10)
pp.predvsobs(pls_raman_calibration,Raman_Spectra_snv_savgol,API_Conc,CLASSID=Tablet_Categories,colorby='Type')
pp.predvsobs(pls_raman_calibration,Raman_Spectra_snv_savgol,API_Conc,CLASSID=Tablet_Categories,colorby='Scale')
def CV (X,Y,BLOCK,ITER,LV):
RMSECV=np.zeros((ITER,LV))
for i in range(ITER):
A=range(X.shape[0])
B=list(A)
random.shuffle(B)
X=X[B,:]
Y=Y[B,:]
C=int(X.shape[0]/BLOCK)
Yval_pred=np.zeros((X.shape[0],LV))
for r in range(BLOCK):
if r==0:
BGN_v=0
END_v=C
D=range(X.shape[0])
E=list(D)
c_=E[C+1:X.shape[0]:1]
# print('A')
elif r==BLOCK-1:
BGN_v=C*r
END_v=X.shape[0]
D=range(X.shape[0])
E=list(D)
c_=E[0:C*r:1]
# print('hehe')
else:
BGN_v=C*r
END_v=C*(r+1)
D=range(X.shape[0])
E=list(D)
temp_1=E[0:C*r:1]
temp_2=E[C*(r+1)+1:X.shape[0]:1]
c_=[*temp_1,*temp_2]
# print('haha')
#
print(BGN_v)
print(END_v)
# print(c_)
Xcal=X[c_,:]
Ycal=Y[c_,:]
Xval=X[BGN_v:END_v,:]
Yval=Y[BGN_v:END_v,:]
for k in range(LV):
pls_calibration=phi.pls(Xcal,Ycal,k+1,mcsX='center',mcsY=True)
yhatval_pls = phi.pls_pred(Xval,pls_calibration)
Yval_pred[BGN_v:END_v,k]=yhatval_pls['Yhat'].ravel()
for k in range(LV):
RMSECV[i,k]=np.sqrt(np.sum((Y.ravel()-Yval_pred[:,k])**2)/Y.shape[0])
from matplotlib.pylab import plt
for i in range(RMSECV.shape[0]):
plt.plot(np.arange(1,LV+1,1),RMSECV[i,:])
plt.title("Scree plot", fontsize=16, fontweight='bold')
plt.xlabel("# of LV")
plt.ylabel("RMSECV")
plt.show()
return RMSECV
#Make some plots
pp.r2pv(pcaobj)
pp.score_scatter(pcaobj, [1, 2], CLASSID=Cars_CLASSID, colorby='Cylinders')
pp.score_scatter(pcaobj, [1, 2], CLASSID=Cars_CLASSID, colorby='Origin')
pp.loadings(pcaobj)
pp.weighted_loadings(pcaobj)
pp.diagnostics(pcaobj, score_plot_xydim=[1, 2])
pp.contributions_plot(pcaobj, Cars_Features, 'scores', to_obs=['Car1'])
pp.contributions_plot(pcaobj,
Cars_Features,
'scores',
to_obs=['Car1'],
from_obs=['Car4'])
# Build a PLS model with 3 PC's
plsobj = phi.pls(Cars_Features, Cars_Performance, 3)
# Build a PLS model with 3 PC's, cross validating by elements removing 5% of the data per round
plsobj = phi.pls(Cars_Features, Cars_Performance, 3, cross_val=5)
# Build a PLS model with 3 PC's, cross validating by elements removing 5% of the data per round add crossval of X Space
plsobj = phi.pls(Cars_Features,
Cars_Performance,
3,
cross_val=5,
cross_val_X=True)
#Make some plots
pp.r2pv(plsobj)
pp.score_scatter(plsobj, [1, 2], CLASSID=Cars_CLASSID, colorby='Cylinders')
pp.score_scatter(plsobj, [1, 2],
CLASSID=Cars_CLASSID,
colorby='Origin',
# PRE-PROCESS SPECTRA
#nir_spectra_2_use = ee.snv(nir_spectra)
nir_spectra_2_use, M = ee.savgol(5, 1, 2, nir_spectra)
# Divide the set into Calibration and Validation taking one in every two samples
nir_spectra_2_use_cal = nir_spectra_2_use[::2, :]
nir_spectra_2_use_val = nir_spectra_2_use[1:nir_spectra_2_use.shape[0]:2, :]
Ck_cal = Ck[::2, :]
Ck_val = Ck[1:Ck.shape[0]:2, :]
dose_source_cal = dose_source[::2, :]
dose_source_val = dose_source[1:dose_source.shape[0]:2, :]
# Build a PLS from [Ck_cal|dose_source_cal] --> Spectra to determine
# number of components
pls_obj = phi.pls(np.hstack((Ck_cal, dose_source_cal)), nir_spectra_2_use_cal,
6)
pp.r2pv(pls_obj)
#Chose 3LV's moving forward. Now to determine the # of NCI
eiotpls_obj = eiot.buildwPLS(nir_spectra_2_use_cal,
Ck_cal,
3,
num_si_u=0,
R_ik=dose_source_cal)
#Plot the Lambdas using MATPLOTLIB
fig, ax = plt.subplots()
ax.plot(list(range(1, 11)), eiotpls_obj['lambdas'][0:10], 'ob')
ax.set_title('Lambda plot for Unsupervised EIOT w PLS')
ax.set_ylabel('Eigenvalues of $\epsilon_{ch}$')
plt.show()
Y = np.array(NIRData['Matrix'][:, 0])
Xcal = X[::2, :]
Xval = X[1:X.shape[1]:2, :]
Xcal = phi.snv(Xcal)
Xval = phi.snv(Xval)
Xcal, M = phi.savgol(5, 1, 2, Xcal)
Xval, M = phi.savgol(5, 1, 2, Xval)
pp.plot_spectra(Xcal)
Ycal = Y[::2]
Ycal = np.reshape(Ycal, (len(Ycal), -1))
Yval = Y[1:X.shape[1]:2]
Yval = np.reshape(Yval, (len(Yval), -1))
mvm_pls = phi.pls(Xcal, Ycal, 1, mcsX='center', mcsY='center')
yhatval_pls = phi.pls_pred(Xval, mvm_pls)
yhatcal_pls = phi.pls_pred(Xcal, mvm_pls)
PLSerrrCAL = []
LWPLSerrCAL = []
#This for loop will make predictions for the calibration set using various values for the localization parameter
for loc_param in [5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 100, 250]:
yhatlw = []
for o in list(range(Xcal.shape[0])):
yhatlw_ = phi.lwpls(Xcal[o, :],
loc_param,
mvm_pls,
Xcal,
Ycal,
pcaobj1 = phi.pca(Cars_Features_MD, 3, md_algorithm='nlp')
print(pcaobj1['P'])
pp.score_scatter(pcaobj1, [1, 2], CLASSID=Cars_CLASSID, colorby='Cylinders')
pp.loadings(pcaobj1)
pcaobj2 = phi.pca(Cars_Features_MD, 3)
print(pcaobj2['P'])
pp.score_scatter(pcaobj2, [1, 2], CLASSID=Cars_CLASSID, colorby='Cylinders')
pp.loadings(pcaobj2)
pcaobj3 = phi.pca(Cars_Features, 3)
print(pcaobj3['P'])
pp.score_scatter(pcaobj3, [1, 2], CLASSID=Cars_CLASSID, colorby='Cylinders')
pp.loadings(pcaobj3)
plsobj = phi.pls(Cars_Features, Cars_Performance, 3)
pp.r2pv(plsobj)
pp.score_scatter(plsobj, [1, 2],
CLASSID=Cars_CLASSID,
colorby='Origin',
add_ci=True)
pp.loadings(plsobj)
plsobj_nlp = phi.pls(Cars_Features_MD, Cars_Performance, 3, md_algorithm='nlp')
pp.r2pv(plsobj_nlp)
pp.score_scatter(plsobj_nlp, [1, 2],
CLASSID=Cars_CLASSID,
colorby='Origin',
add_ci=True)
pp.loadings(plsobj_nlp)