def buildwPLS(Dm,R,A,*,num_si_u=0,R_ik=False): """ PLS-EIOT BUILD Routines by Salvador Garcia-Munoz ([email protected], [email protected]) Parameters ---------- Dm : TYPE, Numpy Array. DESCRIPTION. Mixture spectra [o x lambda]. R : TYPE, Numpy Array. DESCRIPTION. Compositional matrix [o x n] with mass/mole/vol fractions. A : TYPE, Integer. DESCRIPTION. Number of latent variables to use in PLS model. num_si_u : TYPE, optional. DESCRIPTION. Number of un-supervised non-chemical interferences, the default is 0. R_ik : TYPE, Numpy Array. DESCRIPTION. Matrix with supervised non-chemical interferences The default is False. Returns ------- eiot_pred_obj a Dictionary with predicions and diagnostics """ rk=R_ik Ck=R num_sI=num_si_u if isinstance(rk,bool): print('Building a PLS based EIOT Unsupervised object') pls_obj=phi.pls(Ck,Dm,A) Dm_hat=phi.pls_pred(Ck,pls_obj) else: print('Building a PLS based EIOT Supervised object') aux=np.hstack((Ck,rk)) pls_obj=phi.pls(aux,Dm,A) Dm_hat=phi.pls_pred(aux,pls_obj) pls_obj=phi.conv_pls_2_eiot(pls_obj,r_length=Ck.shape[1]) Dm_hat=Dm_hat['Yhat'] if num_sI>0: E_ch = Dm-Dm_hat [U,S,Vh] = np.linalg.svd(E_ch) V = Vh.T S_I = V[:,0:num_sI] S_short = S[0:num_sI] r_I = U[:,0:num_sI] * np.tile(S_short,(U.shape[0],1)) SR = E_ch- r_I @ S_I.T SSR = np.sum(SR**2,1,keepdims=1) lambdas = S[num_sI] else: S_I = np.nan SR = Dm-Dm_hat E_ch = SR [U,S,Vh] = np.linalg.svd(E_ch) V = Vh.T lambdas = np.diag(S) lambdas = np.diag(lambdas) SSR = np.sum(SR**2,axis=1) r_I = np.nan if not(isinstance(S_I,float)): S_I=S_I.T eiot_obj = pls_obj eiot_obj['wPLS'] = True eiot_obj['S_I'] = S_I eiot_obj['E_ch'] = E_ch eiot_obj['r_I'] = r_I eiot_obj['SR'] = SR eiot_obj['SSR'] = SSR eiot_obj['num_sI'] = num_sI eiot_obj['num_e_sI'] = 0 eiot_obj['abs_max_exc_ri'] = np.nan eiot_obj['S_E_CONF_INT'] = np.nan #Convert Numpy objects to lists and dict for PYOMO if not(isinstance(S_I,float)): pyo_S_I = ee.np2D2pyomo(eiot_obj['S_I']) #convert numpy to dictionary pyo_K = np.arange(1,eiot_obj['S_I'].shape[0]+1) #index for non-chemical interferences pyo_K = pyo_K.tolist() else: pyo_S_I = np.nan pyo_K = [0] eiot_obj['pyo_K'] = pyo_K eiot_obj['pyo_S_I'] = pyo_S_I eiot_obj['lambdas'] = lambdas return eiot_obj
def build_supervised_(Dm,Ck,Rk,num_e_sI,num_sI_U): Rk_mean = np.mean(Rk,axis=0,keepdims=1) Rk_ = Rk - np.tile(Rk_mean,(Rk.shape[0],1)) Rk_std = np.std(Rk_,axis=0,keepdims=1,ddof=1) Rk_ = Rk_ / np.tile(Rk_std,(Rk.shape[0],1)) Rk_n = Rk_ # Do not scale binary numbers for exclusive signatures if num_e_sI!=0: for i in range(Rk.shape[1]-num_e_sI,Rk.shape[1]): Rk_n[:,i] = Rk[:,i] Rk_mean[0][i] = 0 Rk_std[0][i] = 1 Rk = Rk_n Ck_aug = np.hstack((Ck,Rk)) num_sI = Rk.shape[1]+num_sI_U S_hat = np.linalg.pinv(Ck_aug.T@Ck_aug)@Ck_aug.T@Dm S_E_tmp = S_hat S_I_S = S_hat[Ck.shape[1]:] # Supervised non-chemical interferences S_hat = S_hat[0:Ck.shape[1]] # Apparent pure spectrum for chemical species flag_force_no_exclusives = 0 if num_e_sI==0: S_I_E = np.nan r_I_E = np.nan r_I_S = Rk elif S_I_S.shape[0]==num_e_sI: S_I_E = S_I_S S_I_S = np.nan r_I_S = np.nan r_I_E = Rk elif S_I_S.shape[0]>num_e_sI: S_I_E = S_I_S[-num_e_sI:] S_I_S = S_I_S[0:S_I_S.shape[0]-num_e_sI] r_I_E = Rk[:,Rk.shape[1]-num_e_sI:] r_I_S = Rk[:,0:Rk.shape[1]-num_e_sI] else: S_I_E = np.nan r_I_E = np.nan flag_force_no_exclusives = 1 #if the given number of exclusive non-chem int is > size(Rk,2) E_ch_tmp = Dm- Ck_aug @ S_E_tmp if num_sI_U>0: [U,S,Vh] = np.linalg.svd(E_ch_tmp) V = Vh.T S_I_U = V[:,0:num_sI_U] S_I_U = S_I_U.T S_short = S[0:num_sI_U] r_I_U = U[:,0:num_sI_U] * np.tile(S_short,(U.shape[0],1)) if isinstance(S_I_E,float): S_E = np.vstack((S_hat,S_I_S,S_I_U)) S_I = np.vstack((S_I_S,S_I_U)) r_I = np.hstack((r_I_S,r_I_U)) index_rk_eq = list(range(1,1+S_I_S.shape[0])) index_rk_ex_eq = False else: if isinstance(S_I_S,float): S_E = np.vstack((S_hat,S_I_U,S_I_E)) S_I = np.vstack((S_I_U,S_I_E)) r_I = np.hstack((r_I_U,r_I_E)) index_rk_eq = list(range(S_I_U.shape[0]+1,S_I_U.shape[0]+S_I_E.shape[0]+1 )) index_rk_ex_eq = index_rk_eq else: S_E = np.vstack((S_hat,S_I_S,S_I_U,S_I_E)) S_I = np.vstack((S_I_S,S_I_U,S_I_E)) r_I = np.hstack((r_I_S,r_I_U,r_I_E)) index_rk_eq = list(range(1,1+S_I_S.shape[0])) + list(range(S_I_S.shape[0]+S_I_U.shape[0]+1,S_I_S.shape[0]+S_I_U.shape[0]+S_I_E.shape[0]+1 )) index_rk_ex_eq = list(range(S_I_S.shape[0]+S_I_U.shape[0]+1,S_I_S.shape[0]+S_I_U.shape[0]+S_I_E.shape[0]+1 )) lambdas = S[num_sI] else: S_E = S_E_tmp if isinstance(S_I_E,float): S_I = S_I_S r_I = r_I_S index_rk_ex_eq = False elif isinstance(S_I_S,float): S_I = S_I_E r_I = r_I_E index_rk_ex_eq = list(range(1,1+S_I_E.shape[0])) else: S_I = np.vstack((S_I_S,S_I_E)) r_I = np.hstack((r_I_S,r_I_E)) index_rk_ex_eq = list(range(1+S_I_S.shape[0],1+S_I.shape[0])) index_rk_eq = list(range(1,1+S_I.shape[0])) [U,S,Vh] = np.linalg.svd(E_ch_tmp) V = Vh.T lambdas = S SR = Dm - np.hstack((Ck,r_I)) @ S_E SSR = np.sum(SR**2,axis=1,keepdims=1) if isinstance(S_I_E,float): abs_max_exc_ri = np.nan else: abs_max_exc_ri = (r_I_E.max(axis=0)- r_I_E.min(axis=0))/2 E_ch = Dm - Ck @ S_hat if isinstance(r_I,float): Ck_r_I= Ck else: Ck_r_I= np.hstack((Ck,r_I)) myaux = np.diagflat(np.diag(Ck_r_I.T @ Ck_r_I)) A = np.linalg.pinv(myaux) A_ = np.reshape(np.diag(A),(A.shape[0],1)) e_T_e = np.diag(SR.T @ SR) e_T_e = e_T_e.T # 1.96 = 95 % CI in a t-distribution S_E_CONF_INT = 1.96 * np.sqrt(np.tile(e_T_e,(A_.shape[0],1)) * np.tile(A_,(1,e_T_e.shape[0]))) eiot_obj = {'S_hat' : S_hat} eiot_obj['S_I'] = S_I eiot_obj['S_E'] = S_E eiot_obj['E_ch'] = E_ch eiot_obj['r_I'] = r_I eiot_obj['SR'] = SR eiot_obj['SSR'] = SSR eiot_obj['num_sI'] = num_sI eiot_obj['Rk_mean'] = Rk_mean eiot_obj['Rk_std'] = Rk_std if flag_force_no_exclusives==1: eiot_obj['num_e_sI'] = 0 else: eiot_obj['num_e_sI'] = num_e_sI eiot_obj['abs_max_exc_ri'] = abs_max_exc_ri eiot_obj['S_E_CONF_INT'] = S_E_CONF_INT #Convert Numpy objects to lists and dict for PYOMO pyo_L = np.arange(1,eiot_obj['S_hat'].shape[1]+1) #index for wavenumbers pyo_N = np.arange(1,eiot_obj['S_hat'].shape[0]+1) #index for chemical species pyo_Me= np.arange(eiot_obj['num_sI']-eiot_obj['num_e_sI']+1,eiot_obj['num_sI']+1) pyo_L = pyo_L.tolist() pyo_N = pyo_N.tolist() pyo_Me= pyo_Me.tolist() pyo_S_hat = ee.np2D2pyomo(eiot_obj['S_hat']) #convert numpy to dictionary pyo_S_I = ee.np2D2pyomo(eiot_obj['S_I']) #convert numpy to dictionary pyo_M = np.arange(1,eiot_obj['S_I'].shape[0]+1) #index for non-chemical interferences pyo_M = pyo_M.tolist() if num_e_sI!=0: aux = num_sI-num_e_sI aux_dict=dict(((j+aux+1), abs_max_exc_ri[j]) for j in range(len(abs_max_exc_ri))) eiot_obj['pyo_L'] = pyo_L eiot_obj['pyo_N'] = pyo_N eiot_obj['pyo_M'] = pyo_M eiot_obj['pyo_S_hat'] = pyo_S_hat eiot_obj['pyo_S_I'] = pyo_S_I eiot_obj['pyo_Me'] = pyo_Me eiot_obj['index_rk_eq'] = index_rk_eq eiot_obj['index_rk_ex_eq'] = index_rk_ex_eq eiot_obj['lambdas'] = lambdas if num_e_sI!=0: eiot_obj['pyo_abs_max_exc_ri']=aux_dict return eiot_obj
def build_(Dm,Ck,num_sI): S_hat = np.linalg.pinv(Ck.T@Ck)@Ck.T@Dm Dm_hat = Ck @ S_hat if num_sI>0: E_ch = Dm-Dm_hat [U,S,Vh] = np.linalg.svd(E_ch) V = Vh.T S_I = V[:,0:num_sI] S_short = S[0:num_sI] r_I = U[:,0:num_sI] * np.tile(S_short,(U.shape[0],1)) S_E = np.vstack((S_hat,S_I.T)) SR = Dm-np.hstack((Ck,r_I)) @ S_E SSR = np.sum(SR**2,1,keepdims=1) lambdas = S[num_sI] else: S_I = np.nan S_E = S_hat SR = Dm-Dm_hat E_ch = SR [U,S,Vh] = np.linalg.svd(E_ch) V = Vh.T lambdas = np.diag(S) lambdas = np.diag(lambdas) SSR = np.sum(SR**2,axis=1) r_I = np.nan #Conf. Intervals for S vectors. if isinstance(r_I,float): #equivalent to isnan(r_I) Ck_r_I= Ck else: Ck_r_I= np.hstack((Ck,r_I)) A = np.linalg.pinv(Ck_r_I.T @ Ck_r_I ) A_ = np.reshape(np.diag(A),(A.shape[0],1)) e_T_e = np.diag(SR.T @ SR) e_T_e = e_T_e.T # 1.96 = 95 % CI in a t-distribution S_E_CONF_INT = 1.96 * np.sqrt(np.tile(e_T_e,(A_.shape[0],1)) * np.tile(A_,(1,e_T_e.shape[0]))) if not(isinstance(S_I,float)): S_I=S_I.T eiot_obj = {'S_hat' : S_hat} eiot_obj['S_I'] = S_I eiot_obj['S_E'] = S_E eiot_obj['E_ch'] = E_ch eiot_obj['r_I'] = r_I eiot_obj['SR'] = SR eiot_obj['SSR'] = SSR eiot_obj['num_sI'] = num_sI eiot_obj['num_e_sI'] = 0 eiot_obj['abs_max_exc_ri'] = np.nan eiot_obj['S_E_CONF_INT'] = S_E_CONF_INT #Convert Numpy objects to lists and dict for PYOMO pyo_L = np.arange(1,eiot_obj['S_hat'].shape[1]+1) #index for wavenumbers pyo_N = np.arange(1,eiot_obj['S_hat'].shape[0]+1) #index for chemical species pyo_L = pyo_L.tolist() pyo_N = pyo_N.tolist() pyo_S_hat = ee.np2D2pyomo(eiot_obj['S_hat']) #convert numpy to dictionary if not(isinstance(S_I,float)): pyo_S_I = ee.np2D2pyomo(eiot_obj['S_I']) #convert numpy to dictionary pyo_M = np.arange(1,eiot_obj['S_I'].shape[0]+1) #index for non-chemical interferences pyo_M = pyo_M.tolist() else: pyo_S_I = np.nan pyo_M = [0] eiot_obj['pyo_L'] = pyo_L eiot_obj['pyo_N'] = pyo_N eiot_obj['pyo_M'] = pyo_M eiot_obj['pyo_S_hat'] = pyo_S_hat eiot_obj['pyo_S_I'] = pyo_S_I eiot_obj['index_rk_eq'] = False eiot_obj['index_rk_ex_eq']=False eiot_obj['lambdas'] = lambdas return eiot_obj
#Ck_val = Ck[1:Ck.shape[0]:2,:] #dose_source_cal = dose_source[::2,:] #dose_source_val = dose_source[1:dose_source.shape[0]:2,:] # Build Unsupervised EIOT Model and plot lambdas eiot_obj = eiot.build(nir_spectra_use, Ck) # Add batch-by-batch non-chemical interference eiot_obj['S_I'] = S_I S_E = np.vstack((eiot_obj['S_hat'], eiot_obj['S_I'])) eiot_obj['S_E'] = S_E eiot_obj['num_e_sI'] = 11 eiot_obj['num_sI'] = 11 eiot_obj['pyo_M'] = np.arange(1, eiot_obj['S_I'].shape[0] + 1) eiot_obj['pyo_M'] = eiot_obj['pyo_M'].tolist() eiot_obj['pyo_S_I'] = ee.np2D2pyomo(eiot_obj['S_I']) eiot_obj['pyo_Me'] = np.arange(eiot_obj['num_sI'] - eiot_obj['num_e_sI'] + 1, eiot_obj['num_sI'] + 1) eiot_obj['pyo_Me'] = eiot_obj['pyo_Me'].tolist() eiot_obj['abs_max_exc_ri'] = 2 rhat = [] ri_hat = [] for i in range(spec_test.shape[0]): pred_unsup = eiot.calc(ee.snv(spec_test[i, :]), eiot_obj) rhat.append(pred_unsup['r_hat']) ri_hat.append(pred_unsup['r_I_hat']) ri_hat = np.array(ri_hat) rhat = np.array(rhat) A = list(range(178)) B = list(range(11))