def mmr_eval_angle(yTest,yPred): # accuracy=0.0 # precision=0.0 # recall=0.0 ## (m,n)=yTest.shape cEvaluation=mmr_base_classes.cls_evaluation() xdot=np.sum(yTest*yPred,axis=1) xnormt=np.sqrt(np.sum(yTest**2,axis=1)) xnormt=xnormt+(xnormt==0) xnormp=np.sqrt(np.sum(yPred**2,axis=1)) xnormp=xnormp+(xnormp==0) xangle=xdot/(xnormt*xnormp) iacc=np.where(xangle>1.0) xangle[iacc]=1.0 iacc=np.where(xangle<-1.0) xangle[iacc]=-1.0 xangle=np.arccos(xangle) accuracy=np.sqrt(np.mean(xangle**2)) cEvaluation.accuracy=accuracy cEvaluation.precision=0 cEvaluation.recall=0 cEvaluation.f1=0 cEvaluation.confusion=np.zeros((2,2)) cEvaluation.confusion[0,0]=0 cEvaluation.confusion[0,1]=0 cEvaluation.confusion[1,0]=0 cEvaluation.confusion[1,1]=0 return(cEvaluation)
def mmr_eval_angle(yTest, yPred):
# accuracy=0.0
# precision=0.0
# recall=0.0
## (m,n)=yTest.shape
cEvaluation = mmr_base_classes.cls_evaluation()
xdot = np.sum(yTest * yPred, axis=1)
xnormt = np.sqrt(np.sum(yTest**2, axis=1))
xnormt = xnormt + (xnormt == 0)
xnormp = np.sqrt(np.sum(yPred**2, axis=1))
xnormp = xnormp + (xnormp == 0)
xangle = xdot / (xnormt * xnormp)
iacc = np.where(xangle > 1.0)
xangle[iacc] = 1.0
iacc = np.where(xangle < -1.0)
xangle[iacc] = -1.0
xangle = np.arccos(xangle)
accuracy = np.sqrt(np.mean(xangle**2))
cEvaluation.accuracy = accuracy
cEvaluation.precision = 0
cEvaluation.recall = 0
cEvaluation.f1 = 0
cEvaluation.confusion = np.zeros((2, 2))
cEvaluation.confusion[0, 0] = 0
cEvaluation.confusion[0, 1] = 0
cEvaluation.confusion[1, 0] = 0
cEvaluation.confusion[1, 1] = 0
return (cEvaluation)
def mmr_eval_real(yTest,yPred): ## (m,n)=yTest.shape cEvaluation=mmr_base_classes.cls_evaluation() ## RMSE cEvaluation.accuracy=np.sqrt(np.mean(np.sum((yTest-yPred)**2,axis=1))) cEvaluation.precision=0 cEvaluation.recall=0 cEvaluation.f1=0 cEvaluation.confusion=np.zeros((2,2)) return(cEvaluation)
def mmr_eval_real(yTest, yPred):
## (m,n)=yTest.shape
cEvaluation = mmr_base_classes.cls_evaluation()
## RMSE
cEvaluation.accuracy = np.sqrt(np.mean(np.sum((yTest - yPred)**2, axis=1)))
cEvaluation.precision = 0
cEvaluation.recall = 0
cEvaluation.f1 = 0
cEvaluation.confusion = np.zeros((2, 2))
return (cEvaluation)
def mmr_eval_binvector(yTest,yPred):
accuracy=0.0
precision=0.0
recall=0.0
(m,n)=yTest.shape
cEvaluation=mmr_base_classes.cls_evaluation()
cEvaluation.classconfusion=np.zeros((n,n))
## class confusion
for i in range(m):
for j in range(n):
for k in range(n):
if yTest[i,j]>0 and yPred[i,k]>0:
cEvaluation.classconfusion[k,j]+=1
xtp=(yTest>0)*(yPred>0)
xfp=(yTest<=0)*(yPred>0)
xfn=(yTest>0)*(yPred<=0)
xtn=(yTest<=0)*(yPred<=0)
xctp=np.sum(xtp,axis=0)
xcfp=np.sum(xfp,axis=0)
xcfn=np.sum(xfn,axis=0)
## xctn=np.sum(xtn,axis=0)
cEvaluation.cprecision=np.zeros(n)
cEvaluation.crecall=np.zeros(n)
cEvaluation.cf1=np.zeros(n)
for i in range(n):
if xctp[i]+xcfp[i]>0:
cEvaluation.cprecision[i]=xctp[i]/(xctp[i]+xcfp[i])
else:
cEvaluation.cprecision[i]=0
if xctp[i]+xcfn[i]>0:
cEvaluation.crecall[i]=xctp[i]/(xctp[i]+xcfn[i])
else:
cEvaluation.crecall[i]=0
if cEvaluation.cprecision[i]+cEvaluation.crecall[i]>0:
cEvaluation.cf1[i]=2*cEvaluation.cprecision[i]*cEvaluation.crecall[i] \
/(cEvaluation.cprecision[i]+cEvaluation.crecall[i])
else:
cEvaluation.cf1[i]=0
tp=np.sum(xtp)
fp=np.sum(xfp)
fn=np.sum(xfn)
tn=np.sum(xtn)
for i in range(m):
if np.sum(yTest[i]*yPred[i])==n:
accuracy+=1.0
if tp+fp>0:
precision=tp/(tp+fp)
else:
precision=0
if tp+fn>0:
recall=tp/(tp+fn)
else:
recall=0
if precision+recall>0:
f1=2*precision*recall/(precision+recall)
else:
f1=0
cEvaluation.accuracy=accuracy/m
cEvaluation.precision=precision
cEvaluation.recall=recall
cEvaluation.f1=f1
cEvaluation.confusion=np.zeros((2,2))
cEvaluation.confusion[0,0]=tp
cEvaluation.confusion[0,1]=fp
cEvaluation.confusion[1,0]=fn
cEvaluation.confusion[1,1]=tn
return(cEvaluation)
def mmr_eval_binvector(yTest, yPred):
accuracy = 0.0
precision = 0.0
recall = 0.0
(m, n) = yTest.shape
cEvaluation = mmr_base_classes.cls_evaluation()
cEvaluation.classconfusion = np.zeros((n, n))
## class confusion
for i in range(m):
for j in range(n):
for k in range(n):
if yTest[i, j] > 0 and yPred[i, k] > 0:
cEvaluation.classconfusion[k, j] += 1
xtp = (yTest > 0) * (yPred > 0)
xfp = (yTest <= 0) * (yPred > 0)
xfn = (yTest > 0) * (yPred <= 0)
xtn = (yTest <= 0) * (yPred <= 0)
xctp = np.sum(xtp, axis=0)
xcfp = np.sum(xfp, axis=0)
xcfn = np.sum(xfn, axis=0)
## xctn=np.sum(xtn,axis=0)
cEvaluation.cprecision = np.zeros(n)
cEvaluation.crecall = np.zeros(n)
cEvaluation.cf1 = np.zeros(n)
for i in range(n):
if xctp[i] + xcfp[i] > 0:
cEvaluation.cprecision[i] = xctp[i] / (xctp[i] + xcfp[i])
else:
cEvaluation.cprecision[i] = 0
if xctp[i] + xcfn[i] > 0:
cEvaluation.crecall[i] = xctp[i] / (xctp[i] + xcfn[i])
else:
cEvaluation.crecall[i] = 0
if cEvaluation.cprecision[i] + cEvaluation.crecall[i] > 0:
cEvaluation.cf1[i]=2*cEvaluation.cprecision[i]*cEvaluation.crecall[i] \
/(cEvaluation.cprecision[i]+cEvaluation.crecall[i])
else:
cEvaluation.cf1[i] = 0
tp = np.sum(xtp)
fp = np.sum(xfp)
fn = np.sum(xfn)
tn = np.sum(xtn)
for i in range(m):
if np.sum(yTest[i] * yPred[i]) == n:
accuracy += 1.0
if tp + fp > 0:
precision = tp / (tp + fp)
else:
precision = 0
if tp + fn > 0:
recall = tp / (tp + fn)
else:
recall = 0
if precision + recall > 0:
f1 = 2 * precision * recall / (precision + recall)
else:
f1 = 0
cEvaluation.accuracy = accuracy / m
cEvaluation.precision = precision
cEvaluation.recall = recall
cEvaluation.f1 = f1
cEvaluation.confusion = np.zeros((2, 2))
cEvaluation.confusion[0, 0] = tp
cEvaluation.confusion[0, 1] = fp
cEvaluation.confusion[1, 0] = fn
cEvaluation.confusion[1, 1] = tn
return (cEvaluation)
def mvm_eval(ieval_type,nrow,xdatacls,ZrowT):
"""
Compute the gloabal error measures of the predition
Input:
ieval_type error measures =0 0/1 loss,=1 RMSE, =2 MAE error
nrow number of rows
datacls class of features kernels
ZrowT predicted values, list indexed by row index,
and each list element contains the prediction
of all column elements belonging to that row
Output:
deval accuracy in the corresponding error measure
"""
if xdatacls.testontrain==0:
xranges_tes=xdatacls.xranges_rel_test
xdata_tes=xdatacls.xdata_tes
else:
xranges_tes=xdatacls.xranges_rel
xdata_tes=xdatacls.xdata_tra
txdim=xdata_tes[2].shape
if len(txdim)==1:
nxdim=1
else:
nxdim=txdim[1]
cEval=mmr_base_classes.cls_evaluation()
icandidate_w=0
icandidate_b=0
if ieval_type==0: ## 0/1 loss
Y0=xdatacls.Y0
ncategory=len(Y0)
nall=0
nright=0
tp=0 ## true positive
tn=0 ## true negative
fp=0 ## false positive
fn=0 ## false negative
xworst=10**3
xbest=-xworst
xconfusion=np.zeros((ncategory+1,ncategory+1))
for irow in range(nrow):
if xranges_tes[irow,1]>0:
istart_tes=xranges_tes[irow,0]
nlength_tes=xranges_tes[irow,1]
xobserved=xdata_tes[2][istart_tes:istart_tes+nlength_tes]
xpredicted=ZrowT[irow][0]+0.0
for i in range(nlength_tes):
if xdatacls.category==0 or xdatacls.category==3: ## rank
ipredicted=Y0[np.abs(Y0-xpredicted[i]).argmin()]
else:
ipredicted=xpredicted[i]
xconfusion[xobserved[i],ipredicted]+=1
if xdatacls.ibinary==0:
if ipredicted==xobserved[i]:
nright+=1
else: ## Y0=[-1,+1]
if ipredicted==1:
if xobserved[i]==1:
tp+=1
else:
fp+=1
else:
if xobserved[i]==1:
fn+=1
else:
tn+=1
xconfidence=ZrowT[irow][2][i]
if xconfidence<xworst:
xworst=xconfidence
icandidate_w=istart_tes+i
if xconfidence>xbest:
xbest=xconfidence
icandidate_b=istart_tes+i
nall+=nlength_tes
if nall==0:
nall=1
deval=float(nright)/nall
if xdatacls.ibinary==0:
cEval.accuracy=deval
else:
cEval.accuracy=(tp+tn)/nall
cEval.xconfusion=xconfusion
cEval.deval=cEval.accuracy
if tp+fp>0:
cEval.precision=float(tp)/(tp+fp)
else:
cEval.precision=0.0
if tp+fn>0:
cEval.recall=float(tp)/(tp+fn)
else:
cEval.recall=0.0
if cEval.recall+cEval.precision>0:
cEval.f1=2*cEval.precision*cEval.recall/(cEval.recall+cEval.precision)
else:
cEval.f1=0.0
elif ieval_type==1: # RMSE root mean square error
nall=0
nright=0
xworst=10**3
xbest=-xworst
for irow in range(nrow):
if xranges_tes[irow,1]>0:
istart_tes=xranges_tes[irow,0]
nlength_tes=xranges_tes[irow,1]
nright+=np.sum((ZrowT[irow][0] \
-xdata_tes[2][istart_tes:istart_tes+nlength_tes])**2)
for i in range(nlength_tes):
if nxdim==1:
xconfidence=ZrowT[irow][2][i]**2 ## raw prediction
else:
xconfidence=np.mean(ZrowT[irow][2][i])**2 ## raw prediction
if xconfidence<xworst:
xworst=xconfidence
icandidate_w=istart_tes+i
if xconfidence>xbest:
xbest=xconfidence
icandidate_b=istart_tes+i
nall+=nlength_tes*nxdim
if nall==0:
nall=1
deval=np.sqrt(float(nright)/nall)
cEval.rmse=deval
cEval.deval=deval
elif ieval_type==2: # MAE mean absolute error
nall=0
nright=0
xworst=10**3
xbest=-xworst
lpredict=[]
for irow in range(nrow):
if xranges_tes[irow,1]>0:
istart_tes=xranges_tes[irow,0]
nlength_tes=xranges_tes[irow,1]
## nright+=np.sum(np.abs(np.exp(ZrowT[irow][0]) \
## -np.exp(xdata_tes[2][istart_tes:istart_tes+nlength_tes])))
nright+=np.sum(np.abs(ZrowT[irow][0] \
-xdata_tes[2][istart_tes:istart_tes+nlength_tes]))
for i in range(nlength_tes):
if nxdim==1:
xconfidence=ZrowT[irow][2][i]**2 ## raw prediction
else:
xconfidence=np.mean(ZrowT[irow][2][i])**2 ## raw prediction
if xconfidence<xworst:
xworst=xconfidence
icandidate_w=istart_tes+i
if xconfidence>xbest:
xbest=xconfidence
icandidate_b=istart_tes+i
nall+=nlength_tes*nxdim
if nall==0:
nall=1
deval=float(nright)/nall
cEval.mae=deval
cEval.deval=deval
cEval.accuracy=deval
elif ieval_type==3: # median absolute error
nall=0
nright=0
xworst=10**3
xbest=-xworst
lpredict=[]
for irow in range(nrow):
if xranges_tes[irow,1]>0:
istart_tes=xranges_tes[irow,0]
nlength_tes=xranges_tes[irow,1]
lpredict.extend(np.abs(ZrowT[irow][0] \
-xdata_tes[2][istart_tes:istart_tes+nlength_tes]))
for i in range(nlength_tes):
xconfidence=np.abs(ZrowT[irow][2][i]) ## raw prediction
if xconfidence<xworst:
xworst=xconfidence
icandidate_w=istart_tes+i
if xconfidence>xbest:
xbest=xconfidence
icandidate_b=istart_tes+i
nall+=nlength_tes
if nall==0:
nall=1
deval=np.median(np.array(lpredict))
cEval.mae=deval
cEval.deval=deval
cEval.accuracy=deval
# cEval.xpredict=np.array(lpredict)
if ieval_type==10: ## \{0,1,2,3\}^n
nall=0
nright=0
tp=0 ## true positive
tn=0 ## true negative
fp=0 ## false positive
fn=0 ## false negative
xworst=10**3
xbest=-xworst
ndim=xdatacls.YKernel.ndim
valrange=xdatacls.YKernel.valrange
nval=max(valrange)+1
tdim=[nval]*ndim
xconfusion=np.zeros((ndim,nval,nval))
for irow in range(nrow):
if xranges_tes[irow,1]>0:
istart_tes=xranges_tes[irow,0]
nlength_tes=xranges_tes[irow,1]
xobserved=xdata_tes[2][istart_tes:istart_tes+nlength_tes]
xpredicted=ZrowT[irow][0].astype(int)
ixobserved=np.unravel_index(xobserved,tdim)
ixpredicted=np.unravel_index(xpredicted,tdim)
for i in range(nlength_tes):
for j in range(ndim):
xconfusion[j,ixobserved[j][i],ixpredicted[j][i]]+=1
## !!!!!! should be changed
xconfidence=ZrowT[irow][2][i]
if xconfidence<xworst:
xworst=xconfidence
icandidate_w=istart_tes+i
if xconfidence>xbest:
xbest=xconfidence
icandidate_b=istart_tes+i
nall+=nlength_tes
cEval.accuracy=0
cEval.xconfusion3=xconfusion
ndim=xconfusion.shape[0]
(accuracy_full,accuracy_no0)=confusion_toys(xconfusion)
cEval.accuracy=accuracy_no0[ndim]
cEval.accuracy_full=accuracy_full
cEval.accuracy_no0=accuracy_no0
cEval.deval=cEval.accuracy
if tp+fp>0:
cEval.precision=float(tp)/(tp+fp)
else:
cEval.precision=0.0
if tp+fn>0:
cEval.recall=float(tp)/(tp+fn)
else:
cEval.recall=0.0
if cEval.recall+cEval.precision>0:
cEval.f1=2*cEval.precision*cEval.recall/(cEval.recall+cEval.precision)
else:
cEval.f1=0.0
## sign comparison of the residues of test and prediction
elif ieval_type==11: ## 0/1 loss
Y0=xdatacls.Y0
ncategory=len(Y0)
nall=0
nright=0
tp=0 ## true positive
tn=0 ## true negative
fp=0 ## false positive
fn=0 ## false negative
xworst=10**3
xbest=-xworst
xconfusion=np.zeros((ncategory+1,ncategory+1))
for irow in range(nrow):
if xranges_tes[irow,1]>0:
istart_tes=xranges_tes[irow,0]
nlength_tes=xranges_tes[irow,1]
xobserved=xdata_tes[2][istart_tes:istart_tes+nlength_tes]
## xpredicted=ZrowT[irow][0]+0.0
xobserved=np.sign(xobserved-ZrowT[irow][0]+ZrowT[irow][1])
xpredicted=np.sign(ZrowT[irow][1]+0.0)
for i in range(nlength_tes):
if xdatacls.category==0 or xdatacls.category==3: ## rank
ipredicted=Y0[np.abs(Y0-xpredicted[i]).argmin()]
else:
ipredicted=xpredicted[i]
## we have values -1,0,+1 to make them be correct index 1 is added
xconfusion[xobserved[i]+1,ipredicted+1]+=1
if xdatacls.ibinary==0:
if ipredicted==xobserved[i]:
nright+=1
else: ## Y0=[-1,+1]
if ipredicted==1:
if xobserved[i]==1:
tp+=1
else:
fp+=1
else:
if xobserved[i]==1:
fn+=1
else:
tn+=1
xconfidence=ZrowT[irow][2][i]
if xconfidence<xworst:
xworst=xconfidence
icandidate_w=istart_tes+i
if xconfidence>xbest:
xbest=xconfidence
icandidate_b=istart_tes+i
nall+=nlength_tes
if nall==0:
nall=1
deval=float(nright)/nall
if xdatacls.ibinary==0:
cEval.accuracy=deval
else:
cEval.accuracy=(tp+tn)/nall
cEval.xconfusion=xconfusion
cEval.deval=cEval.accuracy
if tp+fp>0:
cEval.precision=float(tp)/(tp+fp)
else:
cEval.precision=0.0
if tp+fn>0:
cEval.recall=float(tp)/(tp+fn)
else:
cEval.recall=0.0
if cEval.recall+cEval.precision>0:
cEval.f1=2*cEval.precision*cEval.recall/(cEval.recall+cEval.precision)
else:
cEval.f1=0.0
return(cEval,icandidate_w,icandidate_b)