def buildwPLS(Dm,R,A,*,num_si_u=0,R_ik=False):
"""
PLS-EIOT BUILD Routines by Salvador Garcia-Munoz ([email protected], [email protected])
Parameters
----------
Dm : TYPE, Numpy Array.
DESCRIPTION. Mixture spectra [o x lambda].
R : TYPE, Numpy Array.
DESCRIPTION. Compositional matrix [o x n] with mass/mole/vol fractions.
A : TYPE, Integer.
DESCRIPTION. Number of latent variables to use in PLS model.
num_si_u : TYPE, optional.
DESCRIPTION. Number of un-supervised non-chemical interferences, the default is 0.
R_ik : TYPE, Numpy Array.
DESCRIPTION. Matrix with supervised non-chemical interferences The default is False.
Returns
-------
eiot_pred_obj a Dictionary with predicions and diagnostics
"""
rk=R_ik
Ck=R
num_sI=num_si_u
if isinstance(rk,bool):
print('Building a PLS based EIOT Unsupervised object')
pls_obj=phi.pls(Ck,Dm,A)
Dm_hat=phi.pls_pred(Ck,pls_obj)
else:
print('Building a PLS based EIOT Supervised object')
aux=np.hstack((Ck,rk))
pls_obj=phi.pls(aux,Dm,A)
Dm_hat=phi.pls_pred(aux,pls_obj)
pls_obj=phi.conv_pls_2_eiot(pls_obj,r_length=Ck.shape[1])
Dm_hat=Dm_hat['Yhat']
if num_sI>0:
E_ch = Dm-Dm_hat
[U,S,Vh] = np.linalg.svd(E_ch)
V = Vh.T
S_I = V[:,0:num_sI]
S_short = S[0:num_sI]
r_I = U[:,0:num_sI] * np.tile(S_short,(U.shape[0],1))
SR = E_ch- r_I @ S_I.T
SSR = np.sum(SR**2,1,keepdims=1)
lambdas = S[num_sI]
else:
S_I = np.nan
SR = Dm-Dm_hat
E_ch = SR
[U,S,Vh] = np.linalg.svd(E_ch)
V = Vh.T
lambdas = np.diag(S)
lambdas = np.diag(lambdas)
SSR = np.sum(SR**2,axis=1)
r_I = np.nan
if not(isinstance(S_I,float)):
S_I=S_I.T
eiot_obj = pls_obj
eiot_obj['wPLS'] = True
eiot_obj['S_I'] = S_I
eiot_obj['E_ch'] = E_ch
eiot_obj['r_I'] = r_I
eiot_obj['SR'] = SR
eiot_obj['SSR'] = SSR
eiot_obj['num_sI'] = num_sI
eiot_obj['num_e_sI'] = 0
eiot_obj['abs_max_exc_ri'] = np.nan
eiot_obj['S_E_CONF_INT'] = np.nan
#Convert Numpy objects to lists and dict for PYOMO
if not(isinstance(S_I,float)):
pyo_S_I = ee.np2D2pyomo(eiot_obj['S_I']) #convert numpy to dictionary
pyo_K = np.arange(1,eiot_obj['S_I'].shape[0]+1) #index for non-chemical interferences
pyo_K = pyo_K.tolist()
else:
pyo_S_I = np.nan
pyo_K = [0]
eiot_obj['pyo_K'] = pyo_K
eiot_obj['pyo_S_I'] = pyo_S_I
eiot_obj['lambdas'] = lambdas
return eiot_obj
def build_supervised_(Dm,Ck,Rk,num_e_sI,num_sI_U):
Rk_mean = np.mean(Rk,axis=0,keepdims=1)
Rk_ = Rk - np.tile(Rk_mean,(Rk.shape[0],1))
Rk_std = np.std(Rk_,axis=0,keepdims=1,ddof=1)
Rk_ = Rk_ / np.tile(Rk_std,(Rk.shape[0],1))
Rk_n = Rk_
# Do not scale binary numbers for exclusive signatures
if num_e_sI!=0:
for i in range(Rk.shape[1]-num_e_sI,Rk.shape[1]):
Rk_n[:,i] = Rk[:,i]
Rk_mean[0][i] = 0
Rk_std[0][i] = 1
Rk = Rk_n
Ck_aug = np.hstack((Ck,Rk))
num_sI = Rk.shape[1]+num_sI_U
S_hat = np.linalg.pinv(Ck_aug.T@Ck_aug)@Ck_aug.T@Dm
S_E_tmp = S_hat
S_I_S = S_hat[Ck.shape[1]:] # Supervised non-chemical interferences
S_hat = S_hat[0:Ck.shape[1]] # Apparent pure spectrum for chemical species
flag_force_no_exclusives = 0
if num_e_sI==0:
S_I_E = np.nan
r_I_E = np.nan
r_I_S = Rk
elif S_I_S.shape[0]==num_e_sI:
S_I_E = S_I_S
S_I_S = np.nan
r_I_S = np.nan
r_I_E = Rk
elif S_I_S.shape[0]>num_e_sI:
S_I_E = S_I_S[-num_e_sI:]
S_I_S = S_I_S[0:S_I_S.shape[0]-num_e_sI]
r_I_E = Rk[:,Rk.shape[1]-num_e_sI:]
r_I_S = Rk[:,0:Rk.shape[1]-num_e_sI]
else:
S_I_E = np.nan
r_I_E = np.nan
flag_force_no_exclusives = 1 #if the given number of exclusive non-chem int is > size(Rk,2)
E_ch_tmp = Dm- Ck_aug @ S_E_tmp
if num_sI_U>0:
[U,S,Vh] = np.linalg.svd(E_ch_tmp)
V = Vh.T
S_I_U = V[:,0:num_sI_U]
S_I_U = S_I_U.T
S_short = S[0:num_sI_U]
r_I_U = U[:,0:num_sI_U] * np.tile(S_short,(U.shape[0],1))
if isinstance(S_I_E,float):
S_E = np.vstack((S_hat,S_I_S,S_I_U))
S_I = np.vstack((S_I_S,S_I_U))
r_I = np.hstack((r_I_S,r_I_U))
index_rk_eq = list(range(1,1+S_I_S.shape[0]))
index_rk_ex_eq = False
else:
if isinstance(S_I_S,float):
S_E = np.vstack((S_hat,S_I_U,S_I_E))
S_I = np.vstack((S_I_U,S_I_E))
r_I = np.hstack((r_I_U,r_I_E))
index_rk_eq = list(range(S_I_U.shape[0]+1,S_I_U.shape[0]+S_I_E.shape[0]+1 ))
index_rk_ex_eq = index_rk_eq
else:
S_E = np.vstack((S_hat,S_I_S,S_I_U,S_I_E))
S_I = np.vstack((S_I_S,S_I_U,S_I_E))
r_I = np.hstack((r_I_S,r_I_U,r_I_E))
index_rk_eq = list(range(1,1+S_I_S.shape[0])) + list(range(S_I_S.shape[0]+S_I_U.shape[0]+1,S_I_S.shape[0]+S_I_U.shape[0]+S_I_E.shape[0]+1 ))
index_rk_ex_eq = list(range(S_I_S.shape[0]+S_I_U.shape[0]+1,S_I_S.shape[0]+S_I_U.shape[0]+S_I_E.shape[0]+1 ))
lambdas = S[num_sI]
else:
S_E = S_E_tmp
if isinstance(S_I_E,float):
S_I = S_I_S
r_I = r_I_S
index_rk_ex_eq = False
elif isinstance(S_I_S,float):
S_I = S_I_E
r_I = r_I_E
index_rk_ex_eq = list(range(1,1+S_I_E.shape[0]))
else:
S_I = np.vstack((S_I_S,S_I_E))
r_I = np.hstack((r_I_S,r_I_E))
index_rk_ex_eq = list(range(1+S_I_S.shape[0],1+S_I.shape[0]))
index_rk_eq = list(range(1,1+S_I.shape[0]))
[U,S,Vh] = np.linalg.svd(E_ch_tmp)
V = Vh.T
lambdas = S
SR = Dm - np.hstack((Ck,r_I)) @ S_E
SSR = np.sum(SR**2,axis=1,keepdims=1)
if isinstance(S_I_E,float):
abs_max_exc_ri = np.nan
else:
abs_max_exc_ri = (r_I_E.max(axis=0)- r_I_E.min(axis=0))/2
E_ch = Dm - Ck @ S_hat
if isinstance(r_I,float):
Ck_r_I= Ck
else:
Ck_r_I= np.hstack((Ck,r_I))
myaux = np.diagflat(np.diag(Ck_r_I.T @ Ck_r_I))
A = np.linalg.pinv(myaux)
A_ = np.reshape(np.diag(A),(A.shape[0],1))
e_T_e = np.diag(SR.T @ SR)
e_T_e = e_T_e.T
# 1.96 = 95 % CI in a t-distribution
S_E_CONF_INT = 1.96 * np.sqrt(np.tile(e_T_e,(A_.shape[0],1)) * np.tile(A_,(1,e_T_e.shape[0])))
eiot_obj = {'S_hat' : S_hat}
eiot_obj['S_I'] = S_I
eiot_obj['S_E'] = S_E
eiot_obj['E_ch'] = E_ch
eiot_obj['r_I'] = r_I
eiot_obj['SR'] = SR
eiot_obj['SSR'] = SSR
eiot_obj['num_sI'] = num_sI
eiot_obj['Rk_mean'] = Rk_mean
eiot_obj['Rk_std'] = Rk_std
if flag_force_no_exclusives==1:
eiot_obj['num_e_sI'] = 0
else:
eiot_obj['num_e_sI'] = num_e_sI
eiot_obj['abs_max_exc_ri'] = abs_max_exc_ri
eiot_obj['S_E_CONF_INT'] = S_E_CONF_INT
#Convert Numpy objects to lists and dict for PYOMO
pyo_L = np.arange(1,eiot_obj['S_hat'].shape[1]+1) #index for wavenumbers
pyo_N = np.arange(1,eiot_obj['S_hat'].shape[0]+1) #index for chemical species
pyo_Me= np.arange(eiot_obj['num_sI']-eiot_obj['num_e_sI']+1,eiot_obj['num_sI']+1)
pyo_L = pyo_L.tolist()
pyo_N = pyo_N.tolist()
pyo_Me= pyo_Me.tolist()
pyo_S_hat = ee.np2D2pyomo(eiot_obj['S_hat']) #convert numpy to dictionary
pyo_S_I = ee.np2D2pyomo(eiot_obj['S_I']) #convert numpy to dictionary
pyo_M = np.arange(1,eiot_obj['S_I'].shape[0]+1) #index for non-chemical interferences
pyo_M = pyo_M.tolist()
if num_e_sI!=0:
aux = num_sI-num_e_sI
aux_dict=dict(((j+aux+1), abs_max_exc_ri[j]) for j in range(len(abs_max_exc_ri)))
eiot_obj['pyo_L'] = pyo_L
eiot_obj['pyo_N'] = pyo_N
eiot_obj['pyo_M'] = pyo_M
eiot_obj['pyo_S_hat'] = pyo_S_hat
eiot_obj['pyo_S_I'] = pyo_S_I
eiot_obj['pyo_Me'] = pyo_Me
eiot_obj['index_rk_eq'] = index_rk_eq
eiot_obj['index_rk_ex_eq'] = index_rk_ex_eq
eiot_obj['lambdas'] = lambdas
if num_e_sI!=0:
eiot_obj['pyo_abs_max_exc_ri']=aux_dict
return eiot_obj
def build_(Dm,Ck,num_sI):
S_hat = np.linalg.pinv(Ck.T@Ck)@Ck.T@Dm
Dm_hat = Ck @ S_hat
if num_sI>0:
E_ch = Dm-Dm_hat
[U,S,Vh] = np.linalg.svd(E_ch)
V = Vh.T
S_I = V[:,0:num_sI]
S_short = S[0:num_sI]
r_I = U[:,0:num_sI] * np.tile(S_short,(U.shape[0],1))
S_E = np.vstack((S_hat,S_I.T))
SR = Dm-np.hstack((Ck,r_I)) @ S_E
SSR = np.sum(SR**2,1,keepdims=1)
lambdas = S[num_sI]
else:
S_I = np.nan
S_E = S_hat
SR = Dm-Dm_hat
E_ch = SR
[U,S,Vh] = np.linalg.svd(E_ch)
V = Vh.T
lambdas = np.diag(S)
lambdas = np.diag(lambdas)
SSR = np.sum(SR**2,axis=1)
r_I = np.nan
#Conf. Intervals for S vectors.
if isinstance(r_I,float): #equivalent to isnan(r_I)
Ck_r_I= Ck
else:
Ck_r_I= np.hstack((Ck,r_I))
A = np.linalg.pinv(Ck_r_I.T @ Ck_r_I )
A_ = np.reshape(np.diag(A),(A.shape[0],1))
e_T_e = np.diag(SR.T @ SR)
e_T_e = e_T_e.T
# 1.96 = 95 % CI in a t-distribution
S_E_CONF_INT = 1.96 * np.sqrt(np.tile(e_T_e,(A_.shape[0],1)) * np.tile(A_,(1,e_T_e.shape[0])))
if not(isinstance(S_I,float)):
S_I=S_I.T
eiot_obj = {'S_hat' : S_hat}
eiot_obj['S_I'] = S_I
eiot_obj['S_E'] = S_E
eiot_obj['E_ch'] = E_ch
eiot_obj['r_I'] = r_I
eiot_obj['SR'] = SR
eiot_obj['SSR'] = SSR
eiot_obj['num_sI'] = num_sI
eiot_obj['num_e_sI'] = 0
eiot_obj['abs_max_exc_ri'] = np.nan
eiot_obj['S_E_CONF_INT'] = S_E_CONF_INT
#Convert Numpy objects to lists and dict for PYOMO
pyo_L = np.arange(1,eiot_obj['S_hat'].shape[1]+1) #index for wavenumbers
pyo_N = np.arange(1,eiot_obj['S_hat'].shape[0]+1) #index for chemical species
pyo_L = pyo_L.tolist()
pyo_N = pyo_N.tolist()
pyo_S_hat = ee.np2D2pyomo(eiot_obj['S_hat']) #convert numpy to dictionary
if not(isinstance(S_I,float)):
pyo_S_I = ee.np2D2pyomo(eiot_obj['S_I']) #convert numpy to dictionary
pyo_M = np.arange(1,eiot_obj['S_I'].shape[0]+1) #index for non-chemical interferences
pyo_M = pyo_M.tolist()
else:
pyo_S_I = np.nan
pyo_M = [0]
eiot_obj['pyo_L'] = pyo_L
eiot_obj['pyo_N'] = pyo_N
eiot_obj['pyo_M'] = pyo_M
eiot_obj['pyo_S_hat'] = pyo_S_hat
eiot_obj['pyo_S_I'] = pyo_S_I
eiot_obj['index_rk_eq'] = False
eiot_obj['index_rk_ex_eq']=False
eiot_obj['lambdas'] = lambdas
return eiot_obj
#Ck_val = Ck[1:Ck.shape[0]:2,:]
#dose_source_cal = dose_source[::2,:]
#dose_source_val = dose_source[1:dose_source.shape[0]:2,:]
# Build Unsupervised EIOT Model and plot lambdas
eiot_obj = eiot.build(nir_spectra_use, Ck)
# Add batch-by-batch non-chemical interference
eiot_obj['S_I'] = S_I
S_E = np.vstack((eiot_obj['S_hat'], eiot_obj['S_I']))
eiot_obj['S_E'] = S_E
eiot_obj['num_e_sI'] = 11
eiot_obj['num_sI'] = 11
eiot_obj['pyo_M'] = np.arange(1, eiot_obj['S_I'].shape[0] + 1)
eiot_obj['pyo_M'] = eiot_obj['pyo_M'].tolist()
eiot_obj['pyo_S_I'] = ee.np2D2pyomo(eiot_obj['S_I'])
eiot_obj['pyo_Me'] = np.arange(eiot_obj['num_sI'] - eiot_obj['num_e_sI'] + 1,
eiot_obj['num_sI'] + 1)
eiot_obj['pyo_Me'] = eiot_obj['pyo_Me'].tolist()
eiot_obj['abs_max_exc_ri'] = 2
rhat = []
ri_hat = []
for i in range(spec_test.shape[0]):
pred_unsup = eiot.calc(ee.snv(spec_test[i, :]), eiot_obj)
rhat.append(pred_unsup['r_hat'])
ri_hat.append(pred_unsup['r_I_hat'])
ri_hat = np.array(ri_hat)
rhat = np.array(rhat)
A = list(range(178))
B = list(range(11))