def loadData(trainOutput, trainInput, dualParams, kernelParams):
global cMMR
params = mmr_setparams.cls_params()
nview = 1
params.ninputview = nview
cMMR = mmr_mmr_cls.cls_mmr(params.ninputview)
nfold = cMMR.nfold
nrepeat = cMMR.nrepeat
## @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
cdata_store=armar_load_data.cls_label_files(trainOutput,trainInput, \
dualParams,kernelParams)
cdata_store.load_mmr(cMMR)
mdata = cMMR.mdata
## ############################################################
xselector = np.ones(mdata)
xselector[-1] = 0
ifold = 0
cMMR.split_train_test(xselector, ifold)
cMMR.compute_kernels()
## cMMR.Y0=cMMR.YKernel.get_train(cMMR.itrain) ## candidates
cMMR.csolver = mmr_solver_cls.cls_mmr_solver()
def loadData(trainOutput,trainInput,dualParams,kernelParams):
global cMMR
params=mmr_setparams.cls_params()
nview=1
params.ninputview=nview
cMMR=mmr_mmr_cls.cls_mmr(params.ninputview)
nfold=cMMR.nfold
nrepeat=cMMR.nrepeat
## @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
cdata_store=armar_load_data.cls_label_files(trainOutput,trainInput, \
dualParams,kernelParams)
cdata_store.load_mmr(cMMR)
mdata=cMMR.mdata
## ############################################################
xselector=np.ones(mdata)
xselector[-1]=0
ifold=0
cMMR.split_train_test(xselector,ifold)
cMMR.compute_kernels()
## cMMR.Y0=cMMR.YKernel.get_train(cMMR.itrain) ## candidates
cMMR.csolver=mmr_solver_cls.cls_mmr_solver()
def mmr_main(iworkmode):
params = mmr_setparams.cls_params()
np.set_printoptions(precision=4)
dresult = {}
## ---------------------------------------------
list_object = ['parts']
nviews = 1
lfile_in = [[[(3, 4, 21)]]]
tfile_out = (3, 1, 6)
lresult = []
for iobject in range(len(lfile_in)):
tfile_in = lfile_in[iobject]
for ifeature in range(nviews):
params.ninputview = len(tfile_in)
cMMR = mmr_mmr_cls.cls_mmr(params.ninputview)
nfold = cMMR.nfold
nrepeat = cMMR.nrepeat
cMMR.xbias = -0.95 - ifeature * 0.05
print('Xbias:', cMMR.xbias)
nscore = 4
nipar = 1
if cMMR.crossval_mode == 0: ## random
nfold0 = nfold
xresult_test = np.zeros((nipar, nrepeat, nfold0))
xresult_train = np.zeros((nipar, nrepeat, nfold0))
xpr = np.zeros((nipar, nrepeat, nfold0, nscore))
elif cMMR.crossval_mode == 1: ## predefined trianing and test
nrepeat = 1
nfold0 = 1
xresult_test = np.zeros((nipar, nrepeat, nfold0))
xresult_train = np.zeros((nipar, nrepeat, nfold0))
xpr = np.zeros((nipar, nrepeat, nfold0, nscore))
## @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
cdata_store = objpart_load_data.cls_label_files()
cdata_store.load_mmr(cMMR, tfile_in, tfile_out)
mdata = cMMR.mdata
## -----------------------------------------------
print('Output kernel type: ',
cMMR.YKernel.kernel_params.kernel_type)
for i in range(params.ninputview):
print(i, 'Input kernel type: ',
cMMR.XKernel[i].kernel_params.kernel_type)
## -------------------------------------------------
xtime = np.zeros(5)
## ############################################################
nparam = 4 ## C,D,par1,par2
xbest_param = np.zeros((nrepeat, nfold0, nparam))
## xinpar=[0.2,0.3,0.4,0.5,0.6]
for iipar in range(nipar):
print('===================================================')
for irepeat in range(nrepeat):
## split data into training and test
if cMMR.crossval_mode == 0: ## random selection
xselector = np.floor(np.random.random(mdata) * nfold0)
xselector = xselector - (xselector == nfold0)
elif cMMR.crossval_mode == 1: ## preddefined training and test
xselector = np.zeros(mdata)
xselector[cMMR.ifixtrain] = 1
for ifold in range(nfold0):
cMMR.split_train_test(xselector, ifold)
## validation to choose the best parameters
print('Validation')
t0 = time.clock()
## select the kernel to be validated
cMMR.set_validation()
cvalidation = mmr_validation_cls.cls_mmr_validation()
cvalidation.validation_rkernel = cMMR.XKernel[0].title
best_param = cvalidation.mmr_validation(cMMR)
xtime[0] = time.clock() - t0
print('Best parameters found by validation')
print('c: ', best_param.c)
print('d: ', best_param.d)
print('par1: ', best_param.par1)
print('par2: ', best_param.par2)
xbest_param[irepeat, ifold, 0] = best_param.c
xbest_param[irepeat, ifold, 1] = best_param.d
xbest_param[irepeat, ifold, 2] = best_param.par1
xbest_param[irepeat, ifold, 3] = best_param.par2
cMMR.compute_kernels()
cMMR.Y0 = cMMR.YKernel.get_train(
cMMR.itrain) ## candidates
## training with the best parameters
print('Training')
print(cMMR.YKernel.kernel_params.kernel_type, \
cMMR.YKernel.kernel_params.ipar1, \
cMMR.YKernel.kernel_params.ipar2)
for iview in range(cMMR.ninputview):
print(cMMR.XKernel[iview].kernel_params.kernel_type, \
cMMR.XKernel[iview].kernel_params.ipar1, \
cMMR.XKernel[iview].kernel_params.ipar2)
t0 = time.clock()
cOptDual = cMMR.mmr_train()
xtime[1] = time.clock() - t0
## cls transfers the dual variables to the test procedure
## compute tests
## check the train accuracy
print('Test')
cPredictTra = cMMR.mmr_test(cOptDual, itraindata=0)
## counts the proportion the ones predicted correctly
## ######################################
if cMMR.itestmode == 2:
print('Test knn')
ypred=inverse_knn(cMMR.YKernel.get_Y0(cMMR.itrain), \
cPredictTra)
else:
ypred = cPredictTra.zPred
cEvaluationTra= \
mmr_eval_binvector(cMMR.YKernel.get_train(cMMR.itrain), \
ypred)
xresult_train[iipar, irepeat,
ifold] = cEvaluationTra.accuracy
print('>>>>>>>>>>>\n', cEvaluationTra.confusion)
## ######################################
## check the test accuracy
t0 = time.clock()
cPredictTes = cMMR.mmr_test(cOptDual, itraindata=1)
## counts the proportion the ones predicted correctly
if cMMR.itestmode == 2:
ypred=inverse_knn(cMMR.YKernel.get_Y0(cMMR.itrain), \
cPredictTes)
else:
ypred = cPredictTes.zPred
## cEvaluationTes=mmr_eval_binvector(cData.YTest,cPredictTes.zPred)
cEvaluationTes= \
mmr_eval_binvector(cMMR.YKernel.get_test(cMMR.itest), \
ypred)
xtime[2] = time.clock() - t0
xresult_test[iipar, irepeat,
ifold] = cEvaluationTes.accuracy
xpr[iipar, irepeat, ifold,
0] = cEvaluationTes.precision
xpr[iipar, irepeat, ifold, 1] = cEvaluationTes.recall
xpr[iipar, irepeat, ifold, 2] = cEvaluationTes.f1
xpr[iipar, irepeat, ifold, 3] = cEvaluationTes.accuracy
print(cEvaluationTes.confusion)
print(cEvaluationTes.classconfusion)
try:
xclassconfusion += cEvaluationTes.classconfusion
except:
(n, n) = cEvaluationTes.classconfusion.shape
xclassconfusion = np.zeros((n, n))
xclassconfusion += cEvaluationTes.classconfusion
## ####################################
print('Parameter:',iipar,'Repetition: ',irepeat, \
'Fold: ',ifold)
mmr_report('Result on one fold',
xresult_train[iipar,irepeat,ifold], \
xresult_test[iipar,irepeat,ifold], \
xpr[iipar,irepeat,ifold,:])
print(
np.sum(xpr[iipar, irepeat, :ifold + 1, :], 0) /
(ifold + 1))
mmr_report('Result on one repetition',
np.mean(xresult_train[iipar,irepeat,:]), \
np.mean(xresult_test[iipar,irepeat,:]), \
np.mean(xpr[iipar,irepeat,:,:],0))
mmr_report('Result on all repetitions @@@@@@@',
np.mean(xresult_train[iipar,:,:].flatten()), \
np.mean(xresult_test[iipar,:,:].flatten()), \
np.mean(np.mean(xpr[iipar,:,:,:],0),0))
print('Average best parameters')
## sfield=dir(best_param)
xlabels = ('c', 'd', 'par1', 'par2')
for i in range(nparam):
## print(sfield[i])
print(xlabels[i],': ',np.mean(xbest_param[:,:,i]), \
'(',np.std(xbest_param[:,:,i]),')')
print('xtime:', xtime)
sys.stdout.flush()
dresult[ifeature] = (cMMR.xbias,
np.mean(np.mean(xpr[iipar, :, :, :], 0),
0))
for sfeature_type, tresult in dresult.items():
## xhead=cMMR.xbias
headkey = tfile_in[0][0]
xhead = cdata_store.dirvar[headkey][0] + ', ' + cdata_store.dirvar[
headkey][1]
lresult.append((xhead, tresult))
## lresult.sort()
## for litem in lresult:
## print(litem)
print('\\begin{tabular}{l|rrr}')
print('& \\multicolumn{3}{c}{' + 'Objects' + '} \\\\')
print('Feature type & Precision & Recall & F1 \\\\ \\hline')
for litem in lresult:
print(litem[0],' & ','%6.4f'%litem[1][1][0], \
' & ','%6.4f'%litem[1][1][1],' & ','%6.4f'%litem[1][1][2],' \\\\')
print('\\end{tabular}')
print(xclassconfusion)
print('Bye')
return
def test_mvm_main(workmode):
params=mmr_setparams.cls_params()
xdatacls=mvm_mvm_cls.cls_mvm()
nfold=xdatacls.nfold
if xdatacls.itestmode==0:
nfold0=1 ## active learning
else:
nfold0=nfold ## n-fold cross validation
nparacc=2 ## rmse, time
npar=1
xsummary=np.zeros((npar,nparacc))
ifile=0
pselect=0.05
itrates=1
print('ifile:',ifile)
print('itrates:',itrates)
print('pselect:',pselect)
lfiles=[]
for ipar in range(npar):
rmatrix=mvm_random_matrix.cls_label_files()
(xdata,nrow2,ncol2)=rmatrix.load(ifile,pselect,itrain=itrates)
xdatacls.load_data(xdata,xdatacls.categorymax, \
int(nrow2),int(ncol2),None)
scombine=''
if xdatacls.itestmode==0:
if xdatacls.ibootstrap==0:
fname='xresultte_rand'+scombine+'.csv'
elif xdatacls.ibootstrap==1:
fname='xresultte_active'+scombine+'.csv'
elif xdatacls.ibootstrap==2:
fname='xresultte_greedy'+scombine+'.csv'
elif xdatacls.ibootstrap==3:
fname='xresultte_act_rand'+scombine+'.csv'
else:
fname='xresultte_ncross'+scombine+'.csv'
xdatacls.YKernel.ymax=1
# it will be recomputed in mvm_ranges
xdatacls.YKernel.ymin=-1
xdatacls.YKernel.yrange=200 # it will be recomputed in classcol_ranges
xdatacls.YKernel.ystep=(xdatacls.YKernel.ymax-xdatacls.YKernel.ymin) \
/xdatacls.YKernel.yrange
## set_printoptions(precision=4)
nparam=4 # C,D,par1,par2
nreport=4 ## accuracy, precision, recall, f1
xdatacls.prepare_repetition_folding(init_train_size=100)
nrepeat0=xdatacls.nrepeat0
nfold0=xdatacls.nfold0
creport=mmr_report_cls.cls_mmr_report()
creport.create_xaprf(nrepeat=nrepeat0,nfold=nfold0,nreport=nreport)
xbest_param=np.zeros((nrepeat0,nfold0,nparam))
# ############################################################
nval=max(xdatacls.YKernel.valrange)+1
xconfusion3=np.zeros((nrepeat0,nfold0,xdatacls.YKernel.ndim,nval,nval))
xsolvertime=0.0
ireport=0
for irepeat in range(nrepeat0):
xdatacls.prepare_repetition_training()
for ifold in range(nfold0):
xdatacls.prepare_fold_training(ifold)
# validation to choose the best parameters
print('Validation')
xdatacls.set_validation()
cvalidation=mvm_validation_cls.cls_mvm_validation()
cvalidation.validation_rkernel=xdatacls.XKernel[0].title
best_param=cvalidation.mvm_validation(xdatacls)
print('Parameters:',best_param.c,best_param.d, \
best_param.par1,best_param.par2)
print('Best parameters found by validation')
xbest_param[irepeat,ifold,0]=best_param.c
xbest_param[irepeat,ifold,1]=best_param.d
xbest_param[irepeat,ifold,2]=best_param.par1
xbest_param[irepeat,ifold,3]=best_param.par2
# training with the best parameters
print('training')
time0=time.time()
cOptDual= xdatacls.mvm_train()
xsolvertime+=xdatacls.solvertime
print('Training time:',time.time()-time0)
sys.stdout.flush()
# check the train accuracy
print('test on training')
# check the test accuracy
print('test on test')
time0=time.time()
cPredict=xdatacls.mvm_test()
print('Test time:',time.time()-time0)
sys.stdout.flush()
# counts the proportion the ones predicted correctly
# ####################################
time0=time.time()
(cEval,icandidate_w,icandidate_b)=mvm_eval(xdatacls.ieval_type, \
xdatacls.nrow,xdatacls,cPredict.Zrow)
print('Evaluation time:',time.time()-time0)
(qtest,qpred)=makearray(xdatacls,cPredict.Zrow)
if xdatacls.ieval_type==0:
creport.set_xaprf(irepeat,ifold,cEval)
elif xdatacls.ieval_type==10:
creport.set_xaprf(irepeat,ifold,cEval)
xconfusion3[irepeat,ifold]=cEval.xconfusion3
else:
creport.set_xaprf(irepeat,ifold,cEval)
xdatacls.icandidate_w=xdatacls.itest[icandidate_w]
xdatacls.icandidate_b=xdatacls.itest[icandidate_b]
ireport+=1
## print(cEval.xconfusion)
if xdatacls.ieval_type==0:
for xconfrow in cEval.xconfusion:
for ditem in xconfrow:
print('%7.0f'%ditem,end='')
print()
print()
elif xdatacls.ieval_type==10:
for xtable in cEval.xconfusion3:
xsum=np.sum(xtable)
if xsum==0:
xsum=1
xtable=100*xtable/xsum
for xconfrow in xtable:
for ditem in xconfrow:
print('%9.4f'%ditem,end='')
print()
print()
print()
# ####################################
print('*** ipar, repeatation, fold ***')
print(ipar,irepeat,ifold)
if xdatacls.itestmode==1: ## n-fold crossvalidation
creport.report_prf(xmask=[irepeat,ifold], \
stitle='Result in one fold and one repetation', \
ssubtitle='Accuracy on test')
creport.report_prf(xmask=[irepeat,None], \
stitle='Result in one repetation', \
ssubtitle='Mean and std of the accuracy on test')
sys.stdout.flush()
if xdatacls.itestmode==0: ## n-fold crossvalidation
np.savetxt(fname,creport.xresulttes[:ireport,0,:],delimiter=',', \
fmt='%6.4f')
else:
if xdatacls.ieval_type==0:
np.savetxt(fname,np.squeeze(creport.xaprf),delimiter=',', \
fmt='%6.4f')
else:
np.savetxt(fname,creport.xaprf[:,:,0],delimiter=',',fmt='%6.4f')
(xmean,xstd)=creport.report_prf(xmask=[None,None], \
stitle='***** Overall result ****', \
ssubtitle='Mean and std of the accuracy on test + error')
xsummary[ipar,0]=xmean[0]
xsummary[ipar,1]=xsolvertime/(nrepeat0*nfold0)
if xdatacls.ieval_type==10:
confusion_latex(xconfusion3,lfiles)
print('Average best parameters')
xlabels=('c','d','par1','par2')
for i in range(nparam):
print(xlabels[i],': ',np.mean(xbest_param[:,:,i]), \
'(',np.std(xbest_param[:,:,i]),')')
print('$$$$$$$$$ Summary results:')
(m,n)=xsummary.shape
for i in range(m):
for j in range(n):
print('%10.4f'%xsummary[i,j],end='')
print()
## np.savetxt(fname,xresultte[:ireport,0,:],delimiter=',',fmt='%6.4f')
print('Bye')
return
def mmr_main(iworkmode):
params=mmr_setparams.cls_params()
np.set_printoptions(precision=4)
dresult={}
## ---------------------------------------------
list_object=['parts']
nviews=1
lfile_in=[[[(3,4,21)]]]
tfile_out=(3,1,6)
lresult=[]
for iobject in range(len(lfile_in)):
tfile_in=lfile_in[iobject]
for ifeature in range(nviews):
params.ninputview=len(tfile_in)
cMMR=mmr_mmr_cls.cls_mmr(params.ninputview)
nfold=cMMR.nfold
nrepeat=cMMR.nrepeat
cMMR.xbias=-0.95-ifeature*0.05
print('Xbias:',cMMR.xbias)
nscore=4
nipar=1
if cMMR.crossval_mode==0: ## random
nfold0=nfold
xresult_test=np.zeros((nipar,nrepeat,nfold0))
xresult_train=np.zeros((nipar,nrepeat,nfold0))
xpr=np.zeros((nipar,nrepeat,nfold0,nscore))
elif cMMR.crossval_mode==1: ## predefined trianing and test
nrepeat=1
nfold0=1
xresult_test=np.zeros((nipar,nrepeat,nfold0))
xresult_train=np.zeros((nipar,nrepeat,nfold0))
xpr=np.zeros((nipar,nrepeat,nfold0,nscore))
## @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
cdata_store=objpart_load_data.cls_label_files()
cdata_store.load_mmr(cMMR,tfile_in,tfile_out)
mdata=cMMR.mdata
## -----------------------------------------------
print('Output kernel type: ',cMMR.YKernel.kernel_params.kernel_type)
for i in range(params.ninputview):
print(i,'Input kernel type: ',cMMR.XKernel[i].kernel_params.kernel_type)
## -------------------------------------------------
xtime=np.zeros(5)
## ############################################################
nparam=4 ## C,D,par1,par2
xbest_param=np.zeros((nrepeat,nfold0,nparam))
## xinpar=[0.2,0.3,0.4,0.5,0.6]
for iipar in range(nipar):
print('===================================================')
for irepeat in range(nrepeat):
## split data into training and test
if cMMR.crossval_mode==0: ## random selection
xselector=np.floor(np.random.random(mdata)*nfold0)
xselector=xselector-(xselector==nfold0)
elif cMMR.crossval_mode==1: ## preddefined training and test
xselector=np.zeros(mdata)
xselector[cMMR.ifixtrain]=1
for ifold in range(nfold0):
cMMR.split_train_test(xselector,ifold)
## validation to choose the best parameters
print('Validation')
t0=time.clock()
## select the kernel to be validated
cMMR.set_validation()
cvalidation=mmr_validation_cls.cls_mmr_validation()
cvalidation.validation_rkernel=cMMR.XKernel[0].title
best_param=cvalidation.mmr_validation(cMMR)
xtime[0]=time.clock()-t0
print('Best parameters found by validation')
print('c: ',best_param.c)
print('d: ',best_param.d)
print('par1: ',best_param.par1)
print('par2: ',best_param.par2)
xbest_param[irepeat,ifold,0]=best_param.c
xbest_param[irepeat,ifold,1]=best_param.d
xbest_param[irepeat,ifold,2]=best_param.par1
xbest_param[irepeat,ifold,3]=best_param.par2
cMMR.compute_kernels()
cMMR.Y0=cMMR.YKernel.get_train(cMMR.itrain) ## candidates
## training with the best parameters
print('Training')
print(cMMR.YKernel.kernel_params.kernel_type, \
cMMR.YKernel.kernel_params.ipar1, \
cMMR.YKernel.kernel_params.ipar2)
for iview in range(cMMR.ninputview):
print(cMMR.XKernel[iview].kernel_params.kernel_type, \
cMMR.XKernel[iview].kernel_params.ipar1, \
cMMR.XKernel[iview].kernel_params.ipar2)
t0=time.clock()
cOptDual=cMMR.mmr_train()
xtime[1]=time.clock()-t0
## cls transfers the dual variables to the test procedure
## compute tests
## check the train accuracy
print('Test')
cPredictTra=cMMR.mmr_test(cOptDual,itraindata=0)
## counts the proportion the ones predicted correctly
## ######################################
if cMMR.itestmode==2:
print('Test knn')
ypred=inverse_knn(cMMR.YKernel.get_Y0(cMMR.itrain), \
cPredictTra)
else:
ypred=cPredictTra.zPred
cEvaluationTra= \
mmr_eval_binvector(cMMR.YKernel.get_train(cMMR.itrain), \
ypred)
xresult_train[iipar,irepeat,ifold]=cEvaluationTra.accuracy
print('>>>>>>>>>>>\n',cEvaluationTra.confusion)
## ######################################
## check the test accuracy
t0=time.clock()
cPredictTes= cMMR.mmr_test(cOptDual,itraindata=1)
## counts the proportion the ones predicted correctly
if cMMR.itestmode==2:
ypred=inverse_knn(cMMR.YKernel.get_Y0(cMMR.itrain), \
cPredictTes)
else:
ypred=cPredictTes.zPred
## cEvaluationTes=mmr_eval_binvector(cData.YTest,cPredictTes.zPred)
cEvaluationTes= \
mmr_eval_binvector(cMMR.YKernel.get_test(cMMR.itest), \
ypred)
xtime[2]=time.clock()-t0
xresult_test[iipar,irepeat,ifold]=cEvaluationTes.accuracy
xpr[iipar,irepeat,ifold,0]=cEvaluationTes.precision
xpr[iipar,irepeat,ifold,1]=cEvaluationTes.recall
xpr[iipar,irepeat,ifold,2]=cEvaluationTes.f1
xpr[iipar,irepeat,ifold,3]=cEvaluationTes.accuracy
print(cEvaluationTes.confusion)
print(cEvaluationTes.classconfusion)
try:
xclassconfusion+=cEvaluationTes.classconfusion
except:
(n,n)=cEvaluationTes.classconfusion.shape
xclassconfusion=np.zeros((n,n))
xclassconfusion+=cEvaluationTes.classconfusion
## ####################################
print('Parameter:',iipar,'Repetition: ',irepeat, \
'Fold: ',ifold)
mmr_report('Result on one fold',
xresult_train[iipar,irepeat,ifold], \
xresult_test[iipar,irepeat,ifold], \
xpr[iipar,irepeat,ifold,:])
print(np.sum(xpr[iipar,irepeat,:ifold+1,:],0)/(ifold+1))
mmr_report('Result on one repetition',
np.mean(xresult_train[iipar,irepeat,:]), \
np.mean(xresult_test[iipar,irepeat,:]), \
np.mean(xpr[iipar,irepeat,:,:],0))
mmr_report('Result on all repetitions @@@@@@@',
np.mean(xresult_train[iipar,:,:].flatten()), \
np.mean(xresult_test[iipar,:,:].flatten()), \
np.mean(np.mean(xpr[iipar,:,:,:],0),0))
print('Average best parameters')
## sfield=dir(best_param)
xlabels=('c','d','par1','par2')
for i in range(nparam):
## print(sfield[i])
print(xlabels[i],': ',np.mean(xbest_param[:,:,i]), \
'(',np.std(xbest_param[:,:,i]),')')
print('xtime:',xtime)
sys.stdout.flush()
dresult[ifeature]=(cMMR.xbias,np.mean(np.mean(xpr[iipar,:,:,:],0),0))
for sfeature_type,tresult in dresult.items():
## xhead=cMMR.xbias
headkey=tfile_in[0][0]
xhead=cdata_store.dirvar[headkey][0]+', '+cdata_store.dirvar[headkey][1]
lresult.append((xhead,tresult))
## lresult.sort()
## for litem in lresult:
## print(litem)
print('\\begin{tabular}{l|rrr}')
print('& \\multicolumn{3}{c}{'+'Objects'+'} \\\\')
print('Feature type & Precision & Recall & F1 \\\\ \\hline')
for litem in lresult:
print(litem[0],' & ','%6.4f'%litem[1][1][0], \
' & ','%6.4f'%litem[1][1][1],' & ','%6.4f'%litem[1][1][2],' \\\\')
print('\\end{tabular}')
print(xclassconfusion)
print('Bye')
return
def mmr_main(iworkmode, trainingBase, evalFile, performcl):
params = mmr_setparams.cls_params()
## np.set_printoptions(precision=4)
dresult = {}
nview = 1
nobject = 1
## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
## ################################################
lviews = [0, 1] ### this list could contain a subset of 0,1,2
## ################################################
lresult = []
for iobject in range(nobject):
for ifeature in range(nview):
params.ninputview = len(lviews)
cMMR = mmr_mmr_cls.cls_mmr(params.ninputview)
nfold = cMMR.nfold
nrepeat = cMMR.nrepeat
## cMMR.xbias=-0.06 ## 4 categories
cMMR.xbias = 0.02 - ifeature * 0.01
## cMMR.xbias=0.1-ifeature*0.01
nscore = 4
nipar = 1
cMMR.crossval_mode = 1
if cMMR.crossval_mode == 0: ## random
nfold0 = nfold
xresult_test = np.zeros((nipar, nrepeat, nfold0))
xresult_train = np.zeros((nipar, nrepeat, nfold0))
xpr = np.zeros((nipar, nrepeat, nfold0, nscore))
elif cMMR.crossval_mode == 1: ## predefined trianing and test
nrepeat = 1
nfold0 = 1
xresult_test = np.zeros((nipar, nrepeat, nfold0))
xresult_train = np.zeros((nipar, nrepeat, nfold0))
xpr = np.zeros((nipar, nrepeat, nfold0, nscore))
cdata_store = trajlab_load_data.cls_label_files(
trainingBase, evalFile, performcl)
cdata_store.load_mmr(cMMR, lviews)
mdata = cMMR.mdata
xcross = np.zeros((mdata, mdata))
## !!!!!!!!!!!!!!!!!!
## params.validation.rkernel=cMMR.XKernel[0].title
xtime = np.zeros(5)
## ############################################################
nparam = 4 ## C,D,par1,par2
xbest_param = np.zeros((nrepeat, nfold0, nparam))
for iipar in range(nipar):
for irepeat in range(nrepeat):
## split data into training and test
if cMMR.crossval_mode == 0: ## random selection
xselector = np.zeros(mdata)
ifold = 0
for i in range(mdata):
xselector[i] = ifold
ifold += 1
if ifold >= nfold0:
ifold = 0
np.random.shuffle(xselector)
elif cMMR.crossval_mode == 1: ## preddefined training and test
# (added by simon) train with all data but the last one (not elegant, but works)
cMMR.ifixtrain = list(range(mdata - 1))
xselector = np.zeros(mdata)
xselector[cMMR.ifixtrain] = 1
for ifold in range(nfold0):
cMMR.split_train_test(xselector, ifold)
## validation to choose the best parameters
t0 = time.clock()
## !!!!!!!!!!!!!!!!!!!!!!!!!
cMMR.set_validation()
cvalidation = mmr_validation_cls.cls_mmr_validation()
## !!!!!!!!!!!!!!!!!!!!!!!!! no parameter "params"
best_param = cvalidation.mmr_validation(cMMR)
xtime[0] = time.clock() - t0
xbest_param[irepeat, ifold, 0] = best_param.c
xbest_param[irepeat, ifold, 1] = best_param.d
xbest_param[irepeat, ifold, 2] = best_param.par1
xbest_param[irepeat, ifold, 3] = best_param.par2
cMMR.compute_kernels()
cMMR.Y0 = cMMR.YKernel.get_train(
cMMR.itrain) ## candidates
t0 = time.clock()
## !!!!!!!!!!!!!!!!!!!!!!! np "params"
cOptDual = cMMR.mmr_train()
xtime[1] = time.clock() - t0
## cls transfers the dual variables to the test procedure
## compute tests
## check the train accuracy
## !!!!!!!!!!!!!!!!!!!!!!! np "params"
cPredictTra = cMMR.mmr_test(cOptDual, itraindata=0)
## counts the proportion the ones predicted correctly
## ######################################
if cMMR.itestmode == 2:
ypred=inverse_knn(cMMR.YKernel.get_Y0(cMMR.itrain), \
cPredictTra)
else:
ypred = cPredictTra.zPred
cEvaluationTra= \
mmr_eval_binvector(cMMR.YKernel.get_train(cMMR.itrain), \
ypred)
xresult_train[iipar, irepeat,
ifold] = cEvaluationTra.accuracy
## ######################################
## check the test accuracy
t0 = time.clock()
## !!!!!!!!!!!!!!!!!!!!!!! np "params"
cPredictTes = cMMR.mmr_test(cOptDual, itraindata=1)
## counts the proportion the ones predicted correctly
if cMMR.itestmode == 2:
ypred=inverse_knn(cMMR.YKernel.get_Y0(cMMR.itrain), \
cPredictTes)
else:
ypred = cPredictTes.zPred
## cEvaluationTes=mmr_eval_binvector(cData.YTest,cPredictTes.zPred)
cEvaluationTes= \
mmr_eval_binvector(cMMR.YKernel.get_test(cMMR.itest), \
ypred)
xtime[2] = time.clock() - t0
xresult_test[iipar, irepeat,
ifold] = cEvaluationTes.accuracy
xpr[iipar, irepeat, ifold,
0] = cEvaluationTes.precision
xpr[iipar, irepeat, ifold, 1] = cEvaluationTes.recall
xpr[iipar, irepeat, ifold, 2] = cEvaluationTes.f1
xpr[iipar, irepeat, ifold, 3] = cEvaluationTes.accuracy
# (added by simon) for now i will just add the new data to
# the dataset with a random label and check the confusion
# matrix --> very ugly solution but i cant figure it out
# in a clean way
# print(cEvaluationTes.classconfusion)
evaluatedRes = [
row[0] for row in cEvaluationTes.classconfusion
]
evaluatedRes.append(cvalidation.validationScore)
#nonZeroIndexes = [i for i, e in enumerate(evaluatedRes) if e != 0]
#print(evaluatedRes)
#return nonZeroIndexes[0]
return evaluatedRes
try:
xclassconfusion += cEvaluationTes.classconfusion
except:
(n, n) = cEvaluationTes.classconfusion.shape
xclassconfusion = np.zeros((n, n))
xclassconfusion += cEvaluationTes.classconfusion
## mmr_eval_label(ZW,iPre,YTesN,Y0,kit_data,itest,params)
mmr_report.mmr_report('Result on one fold',
xresult_train[iipar,irepeat,ifold], \
xresult_test[iipar,irepeat,ifold], \
xpr[iipar,irepeat,ifold,:])
sys.stdout.flush()
dresult[ifeature] = (cMMR.xbias,
np.mean(np.mean(xpr[iipar, :, :, :], 0),
0))
for sfeature_type, tresult in dresult.items():
## xhead=cMMR.xbias
xhead = ''
lresult.append((xhead, tresult))
return [-1]
def mmr_main(iworkmode):
params=mmr_setparams.cls_params()
np.set_printoptions(precision=4)
dresult={}
## ---------------------------------------------
nview=1
nobject=1
params.ninputview=nview
lresult=[]
for iobject in range(nobject):
for ifeature in range(nview):
cMMR=mmr_mmr_cls.cls_mmr(params.ninputview)
nfold=cMMR.nfold
nrepeat=cMMR.nrepeat
## cMMR.xbias=-0.06 ## 4 categories
cMMR.xbias=0.0
## cMMR.xbias=0.1-ifeature*0.01
print('Xbias:',cMMR.xbias)
nscore=4
nipar=1
if cMMR.crossval_mode==0: ## random
nfold0=nfold
xresult_test=np.zeros((nipar,nrepeat,nfold0))
xresult_train=np.zeros((nipar,nrepeat,nfold0))
xpr=np.zeros((nipar,nrepeat,nfold0,nscore))
elif cMMR.crossval_mode==1: ## predefined trianing and test
nrepeat=1
nfold0=1
xresult_test=np.zeros((nipar,nrepeat,nfold0))
xresult_train=np.zeros((nipar,nrepeat,nfold0))
xpr=np.zeros((nipar,nrepeat,nfold0,nscore))
## @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
## cMMR=mmr_mmr_cls.cls_mmr(params.ninputview)
cdata_store=vision_load_data.cls_label_files()
cdata_store.load_mmr(cMMR)
mdata=cMMR.mdata
## -----------------------------------------------
print('Output kernel type: ',cMMR.YKernel.kernel_params.kernel_type)
for i in range(params.ninputview):
print(i,'Input kernel type: ',cMMR.XKernel[i].kernel_params.kernel_type)
## -------------------------------------------------
xcross=np.zeros((mdata,mdata))
xtime=np.zeros(5)
## ############################################################
nparam=4 ## C,D,par1,par2
xbest_param=np.zeros((nrepeat,nfold0,nparam))
for iipar in range(nipar):
print('===================================================')
for irepeat in range(nrepeat):
## split data into training and test
if cMMR.crossval_mode==0: ## random selection
xselector=np.zeros(mdata)
ifold=0
for i in range(mdata):
xselector[i]=ifold
ifold+=1
if ifold>=nfold0:
ifold=0
np.random.shuffle(xselector)
## xselector=np.floor(np.random.random(mdata)*nfold0)
## xselector=xselector-(xselector==nfold0)
elif cMMR.crossval_mode==1: ## preddefined training and test
xselector=np.zeros(mdata)
xselector[cMMR.ifixtrain]=1
for ifold in range(nfold0):
cMMR.split_train_test(xselector,ifold)
## validation to choose the best parameters
print('Validation')
t0=time.clock()
## select the kernel to be validated
cMMR.set_validation()
cvalidation=mmr_validation_cls.cls_mmr_validation()
cvalidation.validation_rkernel=cMMR.XKernel[0].title
best_param=cvalidation.mmr_validation(cMMR)
xtime[0]=time.clock()-t0
print('Best parameters found by validation')
print('c: ',best_param.c)
print('d: ',best_param.d)
print('par1: ',best_param.par1)
print('par2: ',best_param.par2)
xbest_param[irepeat,ifold,0]=best_param.c
xbest_param[irepeat,ifold,1]=best_param.d
xbest_param[irepeat,ifold,2]=best_param.par1
xbest_param[irepeat,ifold,3]=best_param.par2
cMMR.compute_kernels()
cMMR.Y0=cMMR.YKernel.get_train(cMMR.itrain) ## candidates
## training with the best parameters
print('Training')
print(cMMR.YKernel.kernel_params.kernel_type, \
cMMR.YKernel.kernel_params.ipar1, \
cMMR.YKernel.kernel_params.ipar2)
for iview in range(cMMR.ninputview):
print(cMMR.XKernel[iview].kernel_params.kernel_type, \
cMMR.XKernel[iview].kernel_params.ipar1, \
cMMR.XKernel[iview].kernel_params.ipar2)
t0=time.clock()
cOptDual=cMMR.mmr_train()
xtime[1]=time.clock()-t0
## cls transfers the dual variables to the test procedure
## compute tests
## check the train accuracy
print('Test')
cPredictTra=cMMR.mmr_test(cOptDual,itraindata=0)
## counts the proportion the ones predicted correctly
## ######################################
if cMMR.itestmode==2:
print('Test knn')
ypred=inverse_knn(cMMR.YKernel.get_Y0(cMMR.itrain), \
cPredictTra)
else:
ypred=cPredictTra.zPred
cEvaluationTra= \
mmr_eval_binvector(cMMR.YKernel.get_train(cMMR.itrain), \
ypred)
xresult_train[iipar,irepeat,ifold]=cEvaluationTra.accuracy
print('>>>>>>>>>>>\n',cEvaluationTra.confusion)
## ######################################
## check the test accuracy
t0=time.clock()
cPredictTes= cMMR.mmr_test(cOptDual,itraindata=1)
## counts the proportion the ones predicted correctly
if cMMR.itestmode==2:
ypred=inverse_knn(cMMR.YKernel.get_Y0(cMMR.itrain), \
cPredictTes)
else:
ypred=cPredictTes.zPred
## cEvaluationTes=mmr_eval_binvector(cData.YTest,cPredictTes.zPred)
cEvaluationTes= \
mmr_eval_binvector(cMMR.YKernel.get_test(cMMR.itest), \
ypred)
xtime[2]=time.clock()-t0
xresult_test[iipar,irepeat,ifold]=cEvaluationTes.accuracy
xpr[iipar,irepeat,ifold,0]=cEvaluationTes.precision
xpr[iipar,irepeat,ifold,1]=cEvaluationTes.recall
xpr[iipar,irepeat,ifold,2]=cEvaluationTes.f1
xpr[iipar,irepeat,ifold,3]=cEvaluationTes.accuracy
print(cEvaluationTes.confusion)
print(cEvaluationTes.classconfusion)
try:
xclassconfusion+=cEvaluationTes.classconfusion
except:
(n,n)=cEvaluationTes.classconfusion.shape
xclassconfusion=np.zeros((n,n))
xclassconfusion+=cEvaluationTes.classconfusion
## mmr_eval_label(ZW,iPre,YTesN,Y0,kit_data,itest,params)
## ####################################
print('Parameter:',iipar,'Repetition: ',irepeat, \
'Fold: ',ifold)
mmr_report('Result on one fold',
xresult_train[iipar,irepeat,ifold], \
xresult_test[iipar,irepeat,ifold], \
xpr[iipar,irepeat,ifold,:])
print(np.sum(xpr[iipar,irepeat,:ifold+1,:],0)/(ifold+1))
mmr_report('Result on one repetition',
np.mean(xresult_train[iipar,irepeat,:]), \
np.mean(xresult_test[iipar,irepeat,:]), \
np.mean(xpr[iipar,irepeat,:,:],0))
mmr_report('Result on all repetitions @@@@@@@',
np.mean(xresult_train[iipar,:,:].flatten()), \
np.mean(xresult_test[iipar,:,:].flatten()), \
np.mean(np.mean(xpr[iipar,:,:,:],0),0))
print('Average best parameters')
## sfield=dir(best_param)
xlabels=('c','d','par1','par2')
for i in range(nparam):
## print(sfield[i])
print(xlabels[i],': ',np.mean(xbest_param[:,:,i]), \
'(',np.std(xbest_param[:,:,i]),')')
print('xtime:',xtime)
sys.stdout.flush()
dresult[ifeature]=(cMMR.xbias,np.mean(np.mean(xpr[iipar,:,:,:],0),0))
for sfeature_type,tresult in dresult.items():
## xhead=cMMR.xbias
xhead=''
lresult.append((xhead,tresult))
## lresult.sort()
## for litem in lresult:
## print(litem)
print('\\begin{tabular}{l|rrr}')
print('& \\multicolumn{3}{c}{'+'Objects'+'} \\\\')
print('Feature type & Precision & Recall & F1 \\\\ \\hline')
for litem in lresult:
print(litem[0],' & ','%6.4f'%litem[1][1][0], \
' & ','%6.4f'%litem[1][1][1],' & ','%6.4f'%litem[1][1][2],' \\\\')
print('\\end{tabular}')
## print('\\begin{tabular}{l|rrr}')
## print('& \\multicolumn{3}{c}{'+'Objects'+'} \\\\')
## print('Feature & xbias & Precision & Recall & F1 \\\\ \\hline')
## for litem in lresult:
## print(litem[0],' & ','%6.4f'%litem[1][0],' & ','%6.4f'%litem[1][1][0], \
## ' & ','%6.4f'%litem[1][1][1],' & ','%6.4f'%litem[1][1][2],' \\\\')
## print('\\end{tabular}')
## ##########################################################
## !!!! It saves the optimal dual variables, and optimal, crossvalidated,
## kernel parameters into files given in vision_load_data.
## prepare full training with the best parameters
ifold=0
xselector=np.ones(mdata)
cMMR.split_train_test(xselector,ifold)
best_param=np.array([ np.mean(xbest_param[:,:,i]) for i in range(nparam)])
cMMR.penalty.c=best_param[0]
cMMR.penalty.d=best_param[1]
cMMR.XKernel[0].kernel_params.ipar1=best_param[2]
cMMR.XKernel[0].kernel_params.ipar2=best_param[3]
cMMR.compute_kernels()
cMMR.Y0=cMMR.YKernel.get_train(cMMR.itrain) ## candidates
## training with the best parameters
print('Full training')
cOptDual=cMMR.mmr_train()
np.savetxt(cdata_store.sbasedir+cdata_store.dual_params,cMMR.dual.alpha, \
fmt='%9.4f')
np.savetxt(cdata_store.sbasedir+cdata_store.kernel_params,best_param[2:], \
fmt='%9.4f')
print(xclassconfusion)
print('Bye')
return
def mmr_main(iworkmode, trainingBase, evalFile, performcl):
params=mmr_setparams.cls_params()
## np.set_printoptions(precision=4)
dresult={}
nview=1
nobject=1
## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
## ################################################
lviews=[0,1] ### this list could contain a subset of 0,1,2
## ################################################
lresult=[]
for iobject in range(nobject):
for ifeature in range(nview):
params.ninputview=len(lviews)
cMMR=mmr_mmr_cls.cls_mmr(params.ninputview)
nfold=cMMR.nfold
nrepeat=cMMR.nrepeat
## cMMR.xbias=-0.06 ## 4 categories
cMMR.xbias=0.02-ifeature*0.01
## cMMR.xbias=0.1-ifeature*0.01
nscore=4
nipar=1
cMMR.crossval_mode = 1
if cMMR.crossval_mode==0: ## random
nfold0=nfold
xresult_test=np.zeros((nipar,nrepeat,nfold0))
xresult_train=np.zeros((nipar,nrepeat,nfold0))
xpr=np.zeros((nipar,nrepeat,nfold0,nscore))
elif cMMR.crossval_mode==1: ## predefined trianing and test
nrepeat=1
nfold0=1
xresult_test=np.zeros((nipar,nrepeat,nfold0))
xresult_train=np.zeros((nipar,nrepeat,nfold0))
xpr=np.zeros((nipar,nrepeat,nfold0,nscore))
cdata_store = trajlab_load_data.cls_label_files(trainingBase, evalFile, performcl)
cdata_store.load_mmr(cMMR, lviews)
mdata=cMMR.mdata
xcross=np.zeros((mdata,mdata))
## !!!!!!!!!!!!!!!!!!
## params.validation.rkernel=cMMR.XKernel[0].title
xtime=np.zeros(5)
## ############################################################
nparam=4 ## C,D,par1,par2
xbest_param=np.zeros((nrepeat,nfold0,nparam))
for iipar in range(nipar):
for irepeat in range(nrepeat):
## split data into training and test
if cMMR.crossval_mode==0: ## random selection
xselector=np.zeros(mdata)
ifold=0
for i in range(mdata):
xselector[i]=ifold
ifold+=1
if ifold>=nfold0:
ifold=0
np.random.shuffle(xselector)
elif cMMR.crossval_mode==1: ## preddefined training and test
# (added by simon) train with all data but the last one (not elegant, but works)
cMMR.ifixtrain = list(range(mdata - 1))
xselector = np.zeros(mdata)
xselector[cMMR.ifixtrain] = 1
for ifold in range(nfold0):
cMMR.split_train_test(xselector,ifold)
## validation to choose the best parameters
t0 = time.clock()
## !!!!!!!!!!!!!!!!!!!!!!!!!
cMMR.set_validation()
cvalidation=mmr_validation_cls.cls_mmr_validation()
## !!!!!!!!!!!!!!!!!!!!!!!!! no parameter "params"
best_param = cvalidation.mmr_validation(cMMR)
xtime[0] = time.clock() - t0
xbest_param[irepeat,ifold,0]=best_param.c
xbest_param[irepeat,ifold,1]=best_param.d
xbest_param[irepeat,ifold,2]=best_param.par1
xbest_param[irepeat,ifold,3]=best_param.par2
cMMR.compute_kernels()
cMMR.Y0=cMMR.YKernel.get_train(cMMR.itrain) ## candidates
t0=time.clock()
## !!!!!!!!!!!!!!!!!!!!!!! np "params"
cOptDual=cMMR.mmr_train()
xtime[1]=time.clock()-t0
## cls transfers the dual variables to the test procedure
## compute tests
## check the train accuracy
## !!!!!!!!!!!!!!!!!!!!!!! np "params"
cPredictTra = cMMR.mmr_test(cOptDual,itraindata=0)
## counts the proportion the ones predicted correctly
## ######################################
if cMMR.itestmode==2:
ypred=inverse_knn(cMMR.YKernel.get_Y0(cMMR.itrain), \
cPredictTra)
else:
ypred=cPredictTra.zPred
cEvaluationTra= \
mmr_eval_binvector(cMMR.YKernel.get_train(cMMR.itrain), \
ypred)
xresult_train[iipar,irepeat,ifold]=cEvaluationTra.accuracy
## ######################################
## check the test accuracy
t0=time.clock()
## !!!!!!!!!!!!!!!!!!!!!!! np "params"
cPredictTes = cMMR.mmr_test(cOptDual,itraindata=1)
## counts the proportion the ones predicted correctly
if cMMR.itestmode==2:
ypred=inverse_knn(cMMR.YKernel.get_Y0(cMMR.itrain), \
cPredictTes)
else:
ypred=cPredictTes.zPred
## cEvaluationTes=mmr_eval_binvector(cData.YTest,cPredictTes.zPred)
cEvaluationTes= \
mmr_eval_binvector(cMMR.YKernel.get_test(cMMR.itest), \
ypred)
xtime[2] = time.clock() - t0
xresult_test[iipar,irepeat,ifold] = cEvaluationTes.accuracy
xpr[iipar,irepeat,ifold,0]=cEvaluationTes.precision
xpr[iipar,irepeat,ifold,1]=cEvaluationTes.recall
xpr[iipar,irepeat,ifold,2]=cEvaluationTes.f1
xpr[iipar,irepeat,ifold,3]=cEvaluationTes.accuracy
# (added by simon) for now i will just add the new data to
# the dataset with a random label and check the confusion
# matrix --> very ugly solution but i cant figure it out
# in a clean way
# print(cEvaluationTes.classconfusion)
evaluatedRes = [row[0] for row in cEvaluationTes.classconfusion]
evaluatedRes.append(cvalidation.validationScore)
#nonZeroIndexes = [i for i, e in enumerate(evaluatedRes) if e != 0]
#print(evaluatedRes)
#return nonZeroIndexes[0]
return evaluatedRes
try:
xclassconfusion+=cEvaluationTes.classconfusion
except:
(n,n)=cEvaluationTes.classconfusion.shape
xclassconfusion=np.zeros((n,n))
xclassconfusion+=cEvaluationTes.classconfusion
## mmr_eval_label(ZW,iPre,YTesN,Y0,kit_data,itest,params)
mmr_report.mmr_report('Result on one fold',
xresult_train[iipar,irepeat,ifold], \
xresult_test[iipar,irepeat,ifold], \
xpr[iipar,irepeat,ifold,:])
sys.stdout.flush()
dresult[ifeature]=(cMMR.xbias,np.mean(np.mean(xpr[iipar,:,:,:],0),0))
for sfeature_type,tresult in dresult.items():
## xhead=cMMR.xbias
xhead=''
lresult.append((xhead,tresult))
return [-1]
def roar_main(workmode):
params=mmr_setparams.cls_params()
params.setvalidation()
params.setsolver()
params.setgeneral()
params.setoutput()
params.setinput()
## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
xdatacls=mvm_mvm_cls.cls_mvm()
roar_prepare.roar_prepare(xdatacls)
nfold=xdatacls.nfold
if xdatacls.itestmode in (0,3):
nfold0=1 ## active learning
else:
nfold0=nfold ## n-fold cross validation
nrepeat=xdatacls.nrepeat
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
scombine=''
if xdatacls.itestmode==0:
if xdatacls.ibootstrap==0:
fname='xresultte_rand'+scombine+'.csv'
elif xdatacls.ibootstrap==1:
fname='xresultte_active'+scombine+'.csv'
elif xdatacls.ibootstrap==2:
fname='xresultte_greedy'+scombine+'.csv'
elif xdatacls.ibootstrap==3:
fname='xresultte_act_rand'+scombine+'.csv'
else:
fname='xresultte_ncross'+scombine+'.csv'
## xdatacls.YKernel.ymax=ctables.ncategory
# it will be recomputed in mvm_ranges
xdatacls.YKernel.ymin=0
xdatacls.YKernel.yrange=100 # it will be recomputed in classcol_ranges
xdatacls.YKernel.ystep=1
# load the databases
# data file
ndata=xdatacls.ndata
## set_printoptions(precision=4)
npar=1 ## number of parameter selected for random subsample
nparam=4 # C,D,par1,par2
nreport=4 ## accuracy, precision, recall, f1
if xdatacls.itestmode==0:
nrepeat0=ndata-1 ## active learning
else:
nrepeat0=nrepeat
if xdatacls.itestmode==0:
## initialize the active learning seeds
## pzero=0.001
## xselector=1*(np.random.rand(ndata)<pzero)
nzero=100 ## !!!!!!!! initial training size
xselector=np.zeros(ndata)
nprime=4999
ip=0
for i in range(nzero):
ip+=nprime
if ip>ndata:
ip=ip%ndata
xselector[ip]=1
ndatainit=int(np.sum(xselector))
mtest=ndata-ndatainit
xdatacls.itest=np.where(xselector==0)[0]
icandidate_w=-1
icandidate_b=-1
## nrepeat0=ndata-ndatainit-10
nrepeat0=min(100000,ndata-ndatainit-1000) ## !!!!!! test size
## nrepeat0=1
else: ## n-fold cross validation
nrepeat0=nrepeat
xresulttr=np.zeros((nrepeat0,nfold0))
xresultte=np.zeros((nrepeat0,nfold0,nreport))
xbest_param=np.zeros((nrepeat0,nfold0,nparam))
# ############################################################
# number iterations in the optimization
params.solver.niter=100
print('niter:',params.solver.niter)
for ipar in range(npar):
nval=len(xdatacls.YKernel.valrange)
xconfusion3=np.zeros((nrepeat0,nfold0,xdatacls.YKernel.ndim,nval,nval))
ireport=0
## for irepeat in range(int(float(ndata)/3)):
for irepeat in range(nrepeat0):
if xdatacls.itestmode==0:
if xdatacls.ibootstrap==0:
if icandidate_w>=0:
icandidate_w=np.random.randint(mtest,size=1)
icandidate_w=xdatacls.itest[icandidate_w]
xselector[icandidate_w]=1
## xselector[icandidate_b]=0 ## delete the best
elif xdatacls.ibootstrap==1: ## worst confidence
if icandidate_w>=0:
xselector[icandidate_w]=1
## xselector[icandidate_b]=0 ## delete the best
elif xdatacls.ibootstrap==2: ## best confidence
if icandidate_b>=0:
xselector[icandidate_b]=1
elif xdatacls.ibootstrap==3: ## worst+random
if icandidate_w>=0:
pselect=np.random.rand()
if pselect<0.5:
icandidate_w=np.random.randint(mtest)
icandidate_w=xdatacls.itest[icandidate_w]
xselector[icandidate_w]=1
## xselector[icandidate_b]=0 ## delete the best
elif xdatacls.itestmode==1: ## n-fold cross-validation
## !!! Emre !!!
xselector=np.floor(np.random.random(ndata)*nfold0)
xselector=xselector-(xselector==nfold0)
## if xdatacls.itestmode==1: ## n-fold crossvalidation
## xselector=np.random.randint(nfold0, size=ndata)
## elif xdatacls.itestmode==2: ## random subset
## xselector=1*(np.random.rand(ndata)<float(plist[ipar])/100)
## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
## for test only
elif xdatacls.itestmode==-1:
for i in range(ndata):
xselector[i]=i%nfold0
## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
## xselector_row=np.floor(nfold0*np.random.rand(nrow))
for ifold in range(nfold0):
xdatacls.split_train_test(xselector,ifold)
mtest=len(xdatacls.itest)
if mtest<=0:
print('!!!!!!!')
break
print('mtest:',mtest,'mtrain:',len(xdatacls.itrain))
xdatacls.mvm_datasplit()
# sparse matrices of ranks-row_avarage-col_average+total_avarege
xdatacls.xranges_rel=mvm_ranges(xdatacls.xdata_tra,xdatacls.nrow, \
params)
xdatacls.xranges_rel_test=mvm_ranges(xdatacls.xdata_tes, \
xdatacls.nrow,params)
## mvm_loadmatrix(xdatacls,isubset_tra,params)
if xdatacls.category==0:
mvm_glm(xdatacls,params)
mvm_ygrid(xdatacls,params)
elif xdatacls.category==1:
mvm_largest_category(xdatacls)
elif xdatacls.category==2:
mvm_largest_category(xdatacls)
# validation to choose the best parameters
print('Validation')
xdatacls.set_validation()
params.validation.rkernel=xdatacls.XKernel[0].title
if params.validation.rkernel in xdatacls.dkernels:
kernbest=xdatacls.dkernels[params.validation.rkernel].kernel_params
else:
kernbest=xdatacls.XKernel[0].kernel_params
if params.validation.ivalid==1:
best_param=mvm_validation(xdatacls,params)
else:
best_param=cls_empty_class()
best_param.c=xdatacls.penalty.c
best_param.d=xdatacls.penalty.d
best_param.par1=kernbest.ipar1
best_param.par2=kernbest.ipar2
xdatacls.penalty.c=best_param.c
xdatacls.penalty.d=best_param.d
kernbest.ipar1=best_param.par1
kernbest.ipar2=best_param.par2
print('Parameters:',xdatacls.penalty.c,xdatacls.penalty.d, \
kernbest.ipar1,kernbest.ipar2)
print('Best parameters found by validation')
xbest_param[irepeat,ifold,0]=best_param.c
xbest_param[irepeat,ifold,1]=best_param.d
xbest_param[irepeat,ifold,2]=best_param.par1
xbest_param[irepeat,ifold,3]=best_param.par2
# training with the best parameters
print('training')
time0=time.time()
cOptDual= xdatacls.mvm_train(params)
print('Training time:',time.time()-time0)
# cls transfers the dual variables to the test procedure
# compute test
# check the train accuracy
print('test on training')
# $$$ # counts the proportion the ones predicted correctly
# $$$ # ######################################
# $$$ deval=col_eval(xdatacls.ieval_type,nrow,isubset_tra, \
# $$$ xranges_tra,Zrow)
# $$$ xresulttr(irepeat,ifold)=deval
# ######################################
# check the test accuracy
print('test on test')
time0=time.time()
cPredict=xdatacls.mvm_test(cOptDual.alpha,params)
print('Test time:',time.time()-time0)
# counts the proportion the ones predicted correctly
# ####################################
time0=time.time()
(cEval,icandidate_w,icandidate_b)=mvm_eval(xdatacls.ieval_type, \
xdatacls.nrow,xdatacls,cPredict.Zrow)
print('Evaluation time:',time.time()-time0)
if xdatacls.ieval_type==0:
xresultte[irepeat,ifold,0]=cEval.accuracy
## prediction of effective categories
## part_accuracy=float(np.sum(np.diag(cEval.xconfusion)[1:]))/ \
## np.sum(cEval.xconfusion[1:,1:])
## xresultte[irepeat,ifold,1]=part_accuracy
xresultte[irepeat,ifold,1]=cEval.precision
xresultte[irepeat,ifold,2]=cEval.recall
xresultte[irepeat,ifold,3]=cEval.f1
elif xdatacls.ieval_type==10:
xresultte[irepeat,ifold,0]=cEval.accuracy
xconfusion3[irepeat,ifold]=cEval.xconfusion3
else:
xresultte[irepeat,ifold,0]=cEval.deval
icandidate_w=xdatacls.itest[icandidate_w]
icandidate_b=xdatacls.itest[icandidate_b]
ireport+=1
## print(cEval.xconfusion)
if xdatacls.ieval_type!=10:
for xconfrow in cEval.xconfusion:
for ditem in xconfrow:
print('%7.0f'%ditem,end='')
print()
print()
else:
for xtable in cEval.xconfusion3:
xsum=np.sum(xtable)
if xsum==0:
xsum=1
xtable=100*xtable/xsum
for xconfrow in xtable:
for ditem in xconfrow:
print('%8.4f'%ditem,end='')
print()
print()
print()
# ####################################
print('*** ipar, repeatation, fold ***')
print(ipar,irepeat,ifold)
if xdatacls.itestmode==1: ## n-fold crossvalidation
print('Result in one fold and one repeatation')
## print('Accuracy on train')
## print(xresulttr[irepeat,ifold])
print('Accuracy on test')
if xdatacls.ieval_type==0:
print(xresultte[irepeat,ifold])
else:
print(xresultte[irepeat,ifold,0])
print('Result in one repetation')
print('Mean and std of the accuracy on test')
if xdatacls.ieval_type==0:
print(np.mean(xresultte[irepeat,:,0]),
np.std(xresultte[irepeat,:,0]))
else:
print(np.mean(xresultte[irepeat,:,0]),
np.std(xresultte[irepeat,:,0]))
sys.stdout.flush()
if xdatacls.itestmode==0: ## n-fold crossvalidation
np.savetxt(fname,xresultte[:ireport,0,:],delimiter=',',fmt='%6.4f')
else:
if xdatacls.ieval_type==0:
np.savetxt(fname,xresultte[:ireport,:,:],delimiter=',',fmt='%6.4f')
else:
np.savetxt(fname,xresultte[:ireport,:,0],delimiter=',',fmt='%6.4f')
print('***** Overall result ****')
print('Mean and std of the accuracy on test + error')
if xdatacls.ieval_type==0:
print(np.mean(xresultte[:,:,0]),
np.std(xresultte[:,:,0]))
else:
print(np.mean(xresultte[:,:,0]),
np.std(xresultte[:,:,0]))
# if xdatacls.ieval_type==10:
# confusion_latex(xconfusion3,lfiles)
print('Average best parameters')
## sfield=dir(best_param)
xlabels=('c','d','par1','par2')
for i in range(nparam):
## print(sfield[i])
print(xlabels[i],': ',np.mean(xbest_param[:,:,i]), \
'(',np.std(xbest_param[:,:,i]),')')
## np.savetxt(fname,xresultte[:ireport,0,:],delimiter=',',fmt='%6.4f')
print('Bye')
return
def test_mvm_main(workmode):
params = mmr_setparams.cls_params()
xdatacls = mvm_mvm_cls.cls_mvm()
nfold = xdatacls.nfold
if xdatacls.itestmode == 0:
nfold0 = 1 ## active learning
else:
nfold0 = nfold ## n-fold cross validation
nparacc = 2 ## rmse, time
npar = 1
xsummary = np.zeros((npar, nparacc))
ifile = 0
pselect = 0.05
itrates = 1
print('ifile:', ifile)
print('itrates:', itrates)
print('pselect:', pselect)
lfiles = []
for ipar in range(npar):
rmatrix = mvm_random_matrix.cls_label_files()
(xdata, nrow2, ncol2) = rmatrix.load(ifile, pselect, itrain=itrates)
xdatacls.load_data(xdata,xdatacls.categorymax, \
int(nrow2),int(ncol2),None)
scombine = ''
if xdatacls.itestmode == 0:
if xdatacls.ibootstrap == 0:
fname = 'xresultte_rand' + scombine + '.csv'
elif xdatacls.ibootstrap == 1:
fname = 'xresultte_active' + scombine + '.csv'
elif xdatacls.ibootstrap == 2:
fname = 'xresultte_greedy' + scombine + '.csv'
elif xdatacls.ibootstrap == 3:
fname = 'xresultte_act_rand' + scombine + '.csv'
else:
fname = 'xresultte_ncross' + scombine + '.csv'
xdatacls.YKernel.ymax = 1
# it will be recomputed in mvm_ranges
xdatacls.YKernel.ymin = -1
xdatacls.YKernel.yrange = 200 # it will be recomputed in classcol_ranges
xdatacls.YKernel.ystep=(xdatacls.YKernel.ymax-xdatacls.YKernel.ymin) \
/xdatacls.YKernel.yrange
## set_printoptions(precision=4)
nparam = 4 # C,D,par1,par2
nreport = 4 ## accuracy, precision, recall, f1
xdatacls.prepare_repetition_folding(init_train_size=100)
nrepeat0 = xdatacls.nrepeat0
nfold0 = xdatacls.nfold0
creport = mmr_report_cls.cls_mmr_report()
creport.create_xaprf(nrepeat=nrepeat0, nfold=nfold0, nreport=nreport)
xbest_param = np.zeros((nrepeat0, nfold0, nparam))
# ############################################################
nval = max(xdatacls.YKernel.valrange) + 1
xconfusion3 = np.zeros(
(nrepeat0, nfold0, xdatacls.YKernel.ndim, nval, nval))
xsolvertime = 0.0
ireport = 0
for irepeat in range(nrepeat0):
xdatacls.prepare_repetition_training()
for ifold in range(nfold0):
xdatacls.prepare_fold_training(ifold)
# validation to choose the best parameters
print('Validation')
xdatacls.set_validation()
cvalidation = mvm_validation_cls.cls_mvm_validation()
cvalidation.validation_rkernel = xdatacls.XKernel[0].title
best_param = cvalidation.mvm_validation(xdatacls)
print('Parameters:',best_param.c,best_param.d, \
best_param.par1,best_param.par2)
print('Best parameters found by validation')
xbest_param[irepeat, ifold, 0] = best_param.c
xbest_param[irepeat, ifold, 1] = best_param.d
xbest_param[irepeat, ifold, 2] = best_param.par1
xbest_param[irepeat, ifold, 3] = best_param.par2
# training with the best parameters
print('training')
time0 = time.time()
cOptDual = xdatacls.mvm_train()
xsolvertime += xdatacls.solvertime
print('Training time:', time.time() - time0)
sys.stdout.flush()
# check the train accuracy
print('test on training')
# check the test accuracy
print('test on test')
time0 = time.time()
cPredict = xdatacls.mvm_test()
print('Test time:', time.time() - time0)
sys.stdout.flush()
# counts the proportion the ones predicted correctly
# ####################################
time0 = time.time()
(cEval,icandidate_w,icandidate_b)=mvm_eval(xdatacls.ieval_type, \
xdatacls.nrow,xdatacls,cPredict.Zrow)
print('Evaluation time:', time.time() - time0)
(qtest, qpred) = makearray(xdatacls, cPredict.Zrow)
if xdatacls.ieval_type == 0:
creport.set_xaprf(irepeat, ifold, cEval)
elif xdatacls.ieval_type == 10:
creport.set_xaprf(irepeat, ifold, cEval)
xconfusion3[irepeat, ifold] = cEval.xconfusion3
else:
creport.set_xaprf(irepeat, ifold, cEval)
xdatacls.icandidate_w = xdatacls.itest[icandidate_w]
xdatacls.icandidate_b = xdatacls.itest[icandidate_b]
ireport += 1
## print(cEval.xconfusion)
if xdatacls.ieval_type == 0:
for xconfrow in cEval.xconfusion:
for ditem in xconfrow:
print('%7.0f' % ditem, end='')
print()
print()
elif xdatacls.ieval_type == 10:
for xtable in cEval.xconfusion3:
xsum = np.sum(xtable)
if xsum == 0:
xsum = 1
xtable = 100 * xtable / xsum
for xconfrow in xtable:
for ditem in xconfrow:
print('%9.4f' % ditem, end='')
print()
print()
print()
# ####################################
print('*** ipar, repeatation, fold ***')
print(ipar, irepeat, ifold)
if xdatacls.itestmode == 1: ## n-fold crossvalidation
creport.report_prf(xmask=[irepeat,ifold], \
stitle='Result in one fold and one repetation', \
ssubtitle='Accuracy on test')
creport.report_prf(xmask=[irepeat,None], \
stitle='Result in one repetation', \
ssubtitle='Mean and std of the accuracy on test')
sys.stdout.flush()
if xdatacls.itestmode == 0: ## n-fold crossvalidation
np.savetxt(fname,creport.xresulttes[:ireport,0,:],delimiter=',', \
fmt='%6.4f')
else:
if xdatacls.ieval_type == 0:
np.savetxt(fname,np.squeeze(creport.xaprf),delimiter=',', \
fmt='%6.4f')
else:
np.savetxt(fname,
creport.xaprf[:, :, 0],
delimiter=',',
fmt='%6.4f')
(xmean,xstd)=creport.report_prf(xmask=[None,None], \
stitle='***** Overall result ****', \
ssubtitle='Mean and std of the accuracy on test + error')
xsummary[ipar, 0] = xmean[0]
xsummary[ipar, 1] = xsolvertime / (nrepeat0 * nfold0)
if xdatacls.ieval_type == 10:
confusion_latex(xconfusion3, lfiles)
print('Average best parameters')
xlabels = ('c', 'd', 'par1', 'par2')
for i in range(nparam):
print(xlabels[i],': ',np.mean(xbest_param[:,:,i]), \
'(',np.std(xbest_param[:,:,i]),')')
print('$$$$$$$$$ Summary results:')
(m, n) = xsummary.shape
for i in range(m):
for j in range(n):
print('%10.4f' % xsummary[i, j], end='')
print()
## np.savetxt(fname,xresultte[:ireport,0,:],delimiter=',',fmt='%6.4f')
print('Bye')
return
def roar_main(workmode):
params = mmr_setparams.cls_params()
params.setvalidation()
params.setsolver()
params.setgeneral()
params.setoutput()
params.setinput()
## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
xdatacls = mvm_mvm_cls.cls_mvm()
roar_prepare.roar_prepare(xdatacls)
nfold = xdatacls.nfold
if xdatacls.itestmode in (0, 3):
nfold0 = 1 ## active learning
else:
nfold0 = nfold ## n-fold cross validation
nrepeat = xdatacls.nrepeat
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
scombine = ''
if xdatacls.itestmode == 0:
if xdatacls.ibootstrap == 0:
fname = 'xresultte_rand' + scombine + '.csv'
elif xdatacls.ibootstrap == 1:
fname = 'xresultte_active' + scombine + '.csv'
elif xdatacls.ibootstrap == 2:
fname = 'xresultte_greedy' + scombine + '.csv'
elif xdatacls.ibootstrap == 3:
fname = 'xresultte_act_rand' + scombine + '.csv'
else:
fname = 'xresultte_ncross' + scombine + '.csv'
## xdatacls.YKernel.ymax=ctables.ncategory
# it will be recomputed in mvm_ranges
xdatacls.YKernel.ymin = 0
xdatacls.YKernel.yrange = 100 # it will be recomputed in classcol_ranges
xdatacls.YKernel.ystep = 1
# load the databases
# data file
ndata = xdatacls.ndata
## set_printoptions(precision=4)
npar = 1 ## number of parameter selected for random subsample
nparam = 4 # C,D,par1,par2
nreport = 4 ## accuracy, precision, recall, f1
if xdatacls.itestmode == 0:
nrepeat0 = ndata - 1 ## active learning
else:
nrepeat0 = nrepeat
if xdatacls.itestmode == 0:
## initialize the active learning seeds
## pzero=0.001
## xselector=1*(np.random.rand(ndata)<pzero)
nzero = 100 ## !!!!!!!! initial training size
xselector = np.zeros(ndata)
nprime = 4999
ip = 0
for i in range(nzero):
ip += nprime
if ip > ndata:
ip = ip % ndata
xselector[ip] = 1
ndatainit = int(np.sum(xselector))
mtest = ndata - ndatainit
xdatacls.itest = np.where(xselector == 0)[0]
icandidate_w = -1
icandidate_b = -1
## nrepeat0=ndata-ndatainit-10
nrepeat0 = min(100000, ndata - ndatainit - 1000) ## !!!!!! test size
## nrepeat0=1
else: ## n-fold cross validation
nrepeat0 = nrepeat
xresulttr = np.zeros((nrepeat0, nfold0))
xresultte = np.zeros((nrepeat0, nfold0, nreport))
xbest_param = np.zeros((nrepeat0, nfold0, nparam))
# ############################################################
# number iterations in the optimization
params.solver.niter = 100
print('niter:', params.solver.niter)
for ipar in range(npar):
nval = len(xdatacls.YKernel.valrange)
xconfusion3 = np.zeros(
(nrepeat0, nfold0, xdatacls.YKernel.ndim, nval, nval))
ireport = 0
## for irepeat in range(int(float(ndata)/3)):
for irepeat in range(nrepeat0):
if xdatacls.itestmode == 0:
if xdatacls.ibootstrap == 0:
if icandidate_w >= 0:
icandidate_w = np.random.randint(mtest, size=1)
icandidate_w = xdatacls.itest[icandidate_w]
xselector[icandidate_w] = 1
## xselector[icandidate_b]=0 ## delete the best
elif xdatacls.ibootstrap == 1: ## worst confidence
if icandidate_w >= 0:
xselector[icandidate_w] = 1
## xselector[icandidate_b]=0 ## delete the best
elif xdatacls.ibootstrap == 2: ## best confidence
if icandidate_b >= 0:
xselector[icandidate_b] = 1
elif xdatacls.ibootstrap == 3: ## worst+random
if icandidate_w >= 0:
pselect = np.random.rand()
if pselect < 0.5:
icandidate_w = np.random.randint(mtest)
icandidate_w = xdatacls.itest[icandidate_w]
xselector[icandidate_w] = 1
## xselector[icandidate_b]=0 ## delete the best
elif xdatacls.itestmode == 1: ## n-fold cross-validation
## !!! Emre !!!
xselector = np.floor(np.random.random(ndata) * nfold0)
xselector = xselector - (xselector == nfold0)
## if xdatacls.itestmode==1: ## n-fold crossvalidation
## xselector=np.random.randint(nfold0, size=ndata)
## elif xdatacls.itestmode==2: ## random subset
## xselector=1*(np.random.rand(ndata)<float(plist[ipar])/100)
## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
## for test only
elif xdatacls.itestmode == -1:
for i in range(ndata):
xselector[i] = i % nfold0
## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
## xselector_row=np.floor(nfold0*np.random.rand(nrow))
for ifold in range(nfold0):
xdatacls.split_train_test(xselector, ifold)
mtest = len(xdatacls.itest)
if mtest <= 0:
print('!!!!!!!')
break
print('mtest:', mtest, 'mtrain:', len(xdatacls.itrain))
xdatacls.mvm_datasplit()
# sparse matrices of ranks-row_avarage-col_average+total_avarege
xdatacls.xranges_rel=mvm_ranges(xdatacls.xdata_tra,xdatacls.nrow, \
params)
xdatacls.xranges_rel_test=mvm_ranges(xdatacls.xdata_tes, \
xdatacls.nrow,params)
## mvm_loadmatrix(xdatacls,isubset_tra,params)
if xdatacls.category == 0:
mvm_glm(xdatacls, params)
mvm_ygrid(xdatacls, params)
elif xdatacls.category == 1:
mvm_largest_category(xdatacls)
elif xdatacls.category == 2:
mvm_largest_category(xdatacls)
# validation to choose the best parameters
print('Validation')
xdatacls.set_validation()
params.validation.rkernel = xdatacls.XKernel[0].title
if params.validation.rkernel in xdatacls.dkernels:
kernbest = xdatacls.dkernels[
params.validation.rkernel].kernel_params
else:
kernbest = xdatacls.XKernel[0].kernel_params
if params.validation.ivalid == 1:
best_param = mvm_validation(xdatacls, params)
else:
best_param = cls_empty_class()
best_param.c = xdatacls.penalty.c
best_param.d = xdatacls.penalty.d
best_param.par1 = kernbest.ipar1
best_param.par2 = kernbest.ipar2
xdatacls.penalty.c = best_param.c
xdatacls.penalty.d = best_param.d
kernbest.ipar1 = best_param.par1
kernbest.ipar2 = best_param.par2
print('Parameters:',xdatacls.penalty.c,xdatacls.penalty.d, \
kernbest.ipar1,kernbest.ipar2)
print('Best parameters found by validation')
xbest_param[irepeat, ifold, 0] = best_param.c
xbest_param[irepeat, ifold, 1] = best_param.d
xbest_param[irepeat, ifold, 2] = best_param.par1
xbest_param[irepeat, ifold, 3] = best_param.par2
# training with the best parameters
print('training')
time0 = time.time()
cOptDual = xdatacls.mvm_train(params)
print('Training time:', time.time() - time0)
# cls transfers the dual variables to the test procedure
# compute test
# check the train accuracy
print('test on training')
# $$$ # counts the proportion the ones predicted correctly
# $$$ # ######################################
# $$$ deval=col_eval(xdatacls.ieval_type,nrow,isubset_tra, \
# $$$ xranges_tra,Zrow)
# $$$ xresulttr(irepeat,ifold)=deval
# ######################################
# check the test accuracy
print('test on test')
time0 = time.time()
cPredict = xdatacls.mvm_test(cOptDual.alpha, params)
print('Test time:', time.time() - time0)
# counts the proportion the ones predicted correctly
# ####################################
time0 = time.time()
(cEval,icandidate_w,icandidate_b)=mvm_eval(xdatacls.ieval_type, \
xdatacls.nrow,xdatacls,cPredict.Zrow)
print('Evaluation time:', time.time() - time0)
if xdatacls.ieval_type == 0:
xresultte[irepeat, ifold, 0] = cEval.accuracy
## prediction of effective categories
## part_accuracy=float(np.sum(np.diag(cEval.xconfusion)[1:]))/ \
## np.sum(cEval.xconfusion[1:,1:])
## xresultte[irepeat,ifold,1]=part_accuracy
xresultte[irepeat, ifold, 1] = cEval.precision
xresultte[irepeat, ifold, 2] = cEval.recall
xresultte[irepeat, ifold, 3] = cEval.f1
elif xdatacls.ieval_type == 10:
xresultte[irepeat, ifold, 0] = cEval.accuracy
xconfusion3[irepeat, ifold] = cEval.xconfusion3
else:
xresultte[irepeat, ifold, 0] = cEval.deval
icandidate_w = xdatacls.itest[icandidate_w]
icandidate_b = xdatacls.itest[icandidate_b]
ireport += 1
## print(cEval.xconfusion)
if xdatacls.ieval_type != 10:
for xconfrow in cEval.xconfusion:
for ditem in xconfrow:
print('%7.0f' % ditem, end='')
print()
print()
else:
for xtable in cEval.xconfusion3:
xsum = np.sum(xtable)
if xsum == 0:
xsum = 1
xtable = 100 * xtable / xsum
for xconfrow in xtable:
for ditem in xconfrow:
print('%8.4f' % ditem, end='')
print()
print()
print()
# ####################################
print('*** ipar, repeatation, fold ***')
print(ipar, irepeat, ifold)
if xdatacls.itestmode == 1: ## n-fold crossvalidation
print('Result in one fold and one repeatation')
## print('Accuracy on train')
## print(xresulttr[irepeat,ifold])
print('Accuracy on test')
if xdatacls.ieval_type == 0:
print(xresultte[irepeat, ifold])
else:
print(xresultte[irepeat, ifold, 0])
print('Result in one repetation')
print('Mean and std of the accuracy on test')
if xdatacls.ieval_type == 0:
print(np.mean(xresultte[irepeat, :, 0]),
np.std(xresultte[irepeat, :, 0]))
else:
print(np.mean(xresultte[irepeat, :, 0]),
np.std(xresultte[irepeat, :, 0]))
sys.stdout.flush()
if xdatacls.itestmode == 0: ## n-fold crossvalidation
np.savetxt(fname,
xresultte[:ireport, 0, :],
delimiter=',',
fmt='%6.4f')
else:
if xdatacls.ieval_type == 0:
np.savetxt(fname,
xresultte[:ireport, :, :],
delimiter=',',
fmt='%6.4f')
else:
np.savetxt(fname,
xresultte[:ireport, :, 0],
delimiter=',',
fmt='%6.4f')
print('***** Overall result ****')
print('Mean and std of the accuracy on test + error')
if xdatacls.ieval_type == 0:
print(np.mean(xresultte[:, :, 0]), np.std(xresultte[:, :, 0]))
else:
print(np.mean(xresultte[:, :, 0]), np.std(xresultte[:, :, 0]))
# if xdatacls.ieval_type==10:
# confusion_latex(xconfusion3,lfiles)
print('Average best parameters')
## sfield=dir(best_param)
xlabels = ('c', 'd', 'par1', 'par2')
for i in range(nparam):
## print(sfield[i])
print(xlabels[i],': ',np.mean(xbest_param[:,:,i]), \
'(',np.std(xbest_param[:,:,i]),')')
## np.savetxt(fname,xresultte[:ireport,0,:],delimiter=',',fmt='%6.4f')
print('Bye')
return
def mmr_main(iworkmode):
params=mmr_setparams.cls_params()
## @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
list_features=["annot","DenseHue","DenseHueV3H1", \
"DenseSift","DenseSiftV3H1","Gist", \
"HarrisHue","HarrisHueV3H1","HarrisSift", \
"HarrisSiftV3H1","Hsv","HsvV3H1","Lab", \
"LabV3H1","Rgb","RgbV3H1"]
## data files in the corresponding directories
datadirs=['corel5k','espgame','iaprtc12','mirflickr','pascal07']
## Example lfeatures=[4,8] means we selected the features:
## "DenseSiftV3H1" and "HarrisSift"
lfeatures=[4]
params.ninputview=len(lfeatures)
idata=0 ## process corel5k data set
xdatacls=mmr_mmr_cls.cls_mmr(params.ninputview)
nfold=xdatacls.nfold
nrepeat=xdatacls.nrepeat
print('Xbias:',xdatacls.xbias)
cdata_store=mmr_load_data.cls_data_load()
cdata_store.load_data(xdatacls,idata,lfeatures)
mdata=xdatacls.mdata
## initializing the array collecting the results
nscore=4
nipar=1
if xdatacls.crossval_mode==0: ## random
nfold0=nfold
xresult_test=np.zeros((nipar,nrepeat,nfold0))
xresult_train=np.zeros((nipar,nrepeat,nfold0))
xpr=np.zeros((nipar,nrepeat,nfold0,nscore))
elif xdatacls.crossval_mode==1: ## predefined trianing and test
nrepeat=1
nfold0=1
xresult_test=np.zeros((nipar,nrepeat,nfold0))
xresult_train=np.zeros((nipar,nrepeat,nfold0))
xpr=np.zeros((nipar,nrepeat,nfold0,nscore))
## -----------------------------------------------
print('Output kernel type: ',xdatacls.YKernel.kernel_params.kernel_type)
for i in range(params.ninputview):
print(i,'Input kernel type: ',xdatacls.XKernel[i].kernel_params.kernel_type)
## -------------------------------------------------
xcross=np.zeros((mdata,mdata))
xtime=np.zeros(5)
## ############################################################
nparam=4 ## C,D,par1,par2
xbest_param=np.zeros((nrepeat,nfold0,nparam))
for iipar in range(nipar):
print('===================================================')
for irepeat in range(nrepeat):
xdatacls.prepare_repetition_training(nfold0)
for ifold in range(nfold0):
xdatacls.prepare_fold_training(ifold)
## validation to choose the best parameters
print('Validation')
t0=time.clock()
xdatacls.set_validation()
cvalidation=mmr_validation_cls.cls_mmr_validation()
cvalidation.validation_rkernel=xdatacls.XKernel[0].title
best_param=cvalidation.mmr_validation(xdatacls)
xtime[0]=time.clock()-t0
print('Best parameters found by validation')
print('c: ',best_param.c)
print('d: ',best_param.d)
print('par1: ',best_param.par1)
print('par2: ',best_param.par2)
xbest_param[irepeat,ifold,0]=best_param.c
xbest_param[irepeat,ifold,1]=best_param.d
xbest_param[irepeat,ifold,2]=best_param.par1
xbest_param[irepeat,ifold,3]=best_param.par2
xdatacls.compute_kernels()
xdatacls.Y0=xdatacls.YKernel.get_train(xdatacls.itrain) ## candidates
## training with the best parameters
print('Training')
print(xdatacls.YKernel.kernel_params.kernel_type, \
xdatacls.YKernel.kernel_params.ipar1, \
xdatacls.YKernel.kernel_params.ipar2)
for iview in range(xdatacls.ninputview):
print(xdatacls.XKernel[iview].kernel_params.kernel_type, \
xdatacls.XKernel[iview].kernel_params.ipar1, \
xdatacls.XKernel[iview].kernel_params.ipar2)
t0=time.clock()
cOptDual=xdatacls.mmr_train()
xtime[1]=time.clock()-t0
## cls transfers the dual variables to the test procedure
## compute tests
## check the train accuracy
print('Test')
cPredictTra=xdatacls.mmr_test(cOptDual,itraindata=0)
## counts the proportion the ones predicted correctly
## ######################################
if xdatacls.itestmode==2:
print('Test knn')
ypred=inverse_knn(xdatacls.YKernel.get_Y0(xdatacls.itrain), \
cPredictTra)
else:
ypred=cPredictTra.zPred
cEvaluationTra= \
mmr_eval_binvector(xdatacls.YKernel.get_train(xdatacls.itrain), \
ypred)
xresult_train[iipar,irepeat,ifold]=cEvaluationTra.accuracy
print('>>>>>>>>>>>\n',cEvaluationTra.confusion)
## ######################################
## check the test accuracy
t0=time.clock()
cPredictTes= xdatacls.mmr_test(cOptDual,itraindata=1)
## counts the proportion the ones predicted correctly
if xdatacls.itestmode==2:
ypred=inverse_knn(xdatacls.YKernel.get_Y0(xdatacls.itrain), \
cPredictTes)
else:
ypred=cPredictTes.zPred
## cEvaluationTes=mmr_eval_binvector(cData.YTest,cPredictTes.zPred)
cEvaluationTes= \
mmr_eval_binvector(xdatacls.YKernel.get_test(xdatacls.itest), \
ypred)
xtime[2]=time.clock()-t0
xresult_test[iipar,irepeat,ifold]=cEvaluationTes.accuracy
xpr[iipar,irepeat,ifold,0]=cEvaluationTes.precision
xpr[iipar,irepeat,ifold,1]=cEvaluationTes.recall
xpr[iipar,irepeat,ifold,2]=cEvaluationTes.f1
xpr[iipar,irepeat,ifold,3]=cEvaluationTes.accuracy
print(cEvaluationTes.confusion)
print(cEvaluationTes.classconfusion)
try:
xclassconfusion+=cEvaluationTes.classconfusion
except:
(n,n)=cEvaluationTes.classconfusion.shape
xclassconfusion=np.zeros((n,n))
xclassconfusion+=cEvaluationTes.classconfusion
## mmr_eval_label(ZW,iPre,YTesN,Y0,kit_data,itest,params)
## ####################################
print('Parameter:',iipar,'Repetition: ',irepeat, \
'Fold: ',ifold)
mmr_report.mmr_report('Result on one fold',
xresult_train[iipar,irepeat,ifold], \
xresult_test[iipar,irepeat,ifold], \
xpr[iipar,irepeat,ifold,:])
print(np.sum(xpr[iipar,irepeat,:ifold+1,:],0)/(ifold+1))
mmr_report.mmr_report('Result on one repetition',
np.mean(xresult_train[iipar,irepeat,:]), \
np.mean(xresult_test[iipar,irepeat,:]), \
np.mean(xpr[iipar,irepeat,:,:],0))
mmr_report.mmr_report('Result on all repetitions @@@@@@@',
np.mean(xresult_train[iipar,:,:].flatten()), \
np.mean(xresult_test[iipar,:,:].flatten()), \
np.mean(np.mean(xpr[iipar,:,:,:],0),0))
print('Average best parameters')
xlabels=('c','d','par1','par2')
for i in range(nparam):
print(xlabels[i],': ',np.mean(xbest_param[:,:,i]), \
'(',np.std(xbest_param[:,:,i]),')')
print('xtime:',xtime)
sys.stdout.flush()
print('Bye')
return
def mmr_main(iworkmode):
params = mmr_setparams.cls_params()
np.set_printoptions(precision=4)
dresult = {}
## ---------------------------------------------
nview = 1
nobject = 1
params.ninputview = nview
lresult = []
for iobject in range(nobject):
for ifeature in range(nview):
cMMR = mmr_mmr_cls.cls_mmr(params.ninputview)
nfold = cMMR.nfold
nrepeat = cMMR.nrepeat
## cMMR.xbias=-0.06 ## 4 categories
cMMR.xbias = 0.0
## cMMR.xbias=0.1-ifeature*0.01
print('Xbias:', cMMR.xbias)
nscore = 4
nipar = 1
if cMMR.crossval_mode == 0: ## random
nfold0 = nfold
xresult_test = np.zeros((nipar, nrepeat, nfold0))
xresult_train = np.zeros((nipar, nrepeat, nfold0))
xpr = np.zeros((nipar, nrepeat, nfold0, nscore))
elif cMMR.crossval_mode == 1: ## predefined trianing and test
nrepeat = 1
nfold0 = 1
xresult_test = np.zeros((nipar, nrepeat, nfold0))
xresult_train = np.zeros((nipar, nrepeat, nfold0))
xpr = np.zeros((nipar, nrepeat, nfold0, nscore))
## @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
## cMMR=mmr_mmr_cls.cls_mmr(params.ninputview)
cdata_store = vision_load_data.cls_label_files()
cdata_store.load_mmr(cMMR)
mdata = cMMR.mdata
## -----------------------------------------------
print('Output kernel type: ',
cMMR.YKernel.kernel_params.kernel_type)
for i in range(params.ninputview):
print(i, 'Input kernel type: ',
cMMR.XKernel[i].kernel_params.kernel_type)
## -------------------------------------------------
xcross = np.zeros((mdata, mdata))
xtime = np.zeros(5)
## ############################################################
nparam = 4 ## C,D,par1,par2
xbest_param = np.zeros((nrepeat, nfold0, nparam))
for iipar in range(nipar):
print('===================================================')
for irepeat in range(nrepeat):
## split data into training and test
if cMMR.crossval_mode == 0: ## random selection
xselector = np.zeros(mdata)
ifold = 0
for i in range(mdata):
xselector[i] = ifold
ifold += 1
if ifold >= nfold0:
ifold = 0
np.random.shuffle(xselector)
## xselector=np.floor(np.random.random(mdata)*nfold0)
## xselector=xselector-(xselector==nfold0)
elif cMMR.crossval_mode == 1: ## preddefined training and test
xselector = np.zeros(mdata)
xselector[cMMR.ifixtrain] = 1
for ifold in range(nfold0):
cMMR.split_train_test(xselector, ifold)
## validation to choose the best parameters
print('Validation')
t0 = time.clock()
## select the kernel to be validated
cMMR.set_validation()
cvalidation = mmr_validation_cls.cls_mmr_validation()
cvalidation.validation_rkernel = cMMR.XKernel[0].title
best_param = cvalidation.mmr_validation(cMMR)
xtime[0] = time.clock() - t0
print('Best parameters found by validation')
print('c: ', best_param.c)
print('d: ', best_param.d)
print('par1: ', best_param.par1)
print('par2: ', best_param.par2)
xbest_param[irepeat, ifold, 0] = best_param.c
xbest_param[irepeat, ifold, 1] = best_param.d
xbest_param[irepeat, ifold, 2] = best_param.par1
xbest_param[irepeat, ifold, 3] = best_param.par2
cMMR.compute_kernels()
cMMR.Y0 = cMMR.YKernel.get_train(
cMMR.itrain) ## candidates
## training with the best parameters
print('Training')
print(cMMR.YKernel.kernel_params.kernel_type, \
cMMR.YKernel.kernel_params.ipar1, \
cMMR.YKernel.kernel_params.ipar2)
for iview in range(cMMR.ninputview):
print(cMMR.XKernel[iview].kernel_params.kernel_type, \
cMMR.XKernel[iview].kernel_params.ipar1, \
cMMR.XKernel[iview].kernel_params.ipar2)
t0 = time.clock()
cOptDual = cMMR.mmr_train()
xtime[1] = time.clock() - t0
## cls transfers the dual variables to the test procedure
## compute tests
## check the train accuracy
print('Test')
cPredictTra = cMMR.mmr_test(cOptDual, itraindata=0)
## counts the proportion the ones predicted correctly
## ######################################
if cMMR.itestmode == 2:
print('Test knn')
ypred=inverse_knn(cMMR.YKernel.get_Y0(cMMR.itrain), \
cPredictTra)
else:
ypred = cPredictTra.zPred
cEvaluationTra= \
mmr_eval_binvector(cMMR.YKernel.get_train(cMMR.itrain), \
ypred)
xresult_train[iipar, irepeat,
ifold] = cEvaluationTra.accuracy
print('>>>>>>>>>>>\n', cEvaluationTra.confusion)
## ######################################
## check the test accuracy
t0 = time.clock()
cPredictTes = cMMR.mmr_test(cOptDual, itraindata=1)
## counts the proportion the ones predicted correctly
if cMMR.itestmode == 2:
ypred=inverse_knn(cMMR.YKernel.get_Y0(cMMR.itrain), \
cPredictTes)
else:
ypred = cPredictTes.zPred
## cEvaluationTes=mmr_eval_binvector(cData.YTest,cPredictTes.zPred)
cEvaluationTes= \
mmr_eval_binvector(cMMR.YKernel.get_test(cMMR.itest), \
ypred)
xtime[2] = time.clock() - t0
xresult_test[iipar, irepeat,
ifold] = cEvaluationTes.accuracy
xpr[iipar, irepeat, ifold,
0] = cEvaluationTes.precision
xpr[iipar, irepeat, ifold, 1] = cEvaluationTes.recall
xpr[iipar, irepeat, ifold, 2] = cEvaluationTes.f1
xpr[iipar, irepeat, ifold, 3] = cEvaluationTes.accuracy
print(cEvaluationTes.confusion)
print(cEvaluationTes.classconfusion)
try:
xclassconfusion += cEvaluationTes.classconfusion
except:
(n, n) = cEvaluationTes.classconfusion.shape
xclassconfusion = np.zeros((n, n))
xclassconfusion += cEvaluationTes.classconfusion
## mmr_eval_label(ZW,iPre,YTesN,Y0,kit_data,itest,params)
## ####################################
print('Parameter:',iipar,'Repetition: ',irepeat, \
'Fold: ',ifold)
mmr_report('Result on one fold',
xresult_train[iipar,irepeat,ifold], \
xresult_test[iipar,irepeat,ifold], \
xpr[iipar,irepeat,ifold,:])
print(
np.sum(xpr[iipar, irepeat, :ifold + 1, :], 0) /
(ifold + 1))
mmr_report('Result on one repetition',
np.mean(xresult_train[iipar,irepeat,:]), \
np.mean(xresult_test[iipar,irepeat,:]), \
np.mean(xpr[iipar,irepeat,:,:],0))
mmr_report('Result on all repetitions @@@@@@@',
np.mean(xresult_train[iipar,:,:].flatten()), \
np.mean(xresult_test[iipar,:,:].flatten()), \
np.mean(np.mean(xpr[iipar,:,:,:],0),0))
print('Average best parameters')
## sfield=dir(best_param)
xlabels = ('c', 'd', 'par1', 'par2')
for i in range(nparam):
## print(sfield[i])
print(xlabels[i],': ',np.mean(xbest_param[:,:,i]), \
'(',np.std(xbest_param[:,:,i]),')')
print('xtime:', xtime)
sys.stdout.flush()
dresult[ifeature] = (cMMR.xbias,
np.mean(np.mean(xpr[iipar, :, :, :], 0),
0))
for sfeature_type, tresult in dresult.items():
## xhead=cMMR.xbias
xhead = ''
lresult.append((xhead, tresult))
## lresult.sort()
## for litem in lresult:
## print(litem)
print('\\begin{tabular}{l|rrr}')
print('& \\multicolumn{3}{c}{' + 'Objects' + '} \\\\')
print('Feature type & Precision & Recall & F1 \\\\ \\hline')
for litem in lresult:
print(litem[0],' & ','%6.4f'%litem[1][1][0], \
' & ','%6.4f'%litem[1][1][1],' & ','%6.4f'%litem[1][1][2],' \\\\')
print('\\end{tabular}')
## print('\\begin{tabular}{l|rrr}')
## print('& \\multicolumn{3}{c}{'+'Objects'+'} \\\\')
## print('Feature & xbias & Precision & Recall & F1 \\\\ \\hline')
## for litem in lresult:
## print(litem[0],' & ','%6.4f'%litem[1][0],' & ','%6.4f'%litem[1][1][0], \
## ' & ','%6.4f'%litem[1][1][1],' & ','%6.4f'%litem[1][1][2],' \\\\')
## print('\\end{tabular}')
## ##########################################################
## !!!! It saves the optimal dual variables, and optimal, crossvalidated,
## kernel parameters into files given in vision_load_data.
## prepare full training with the best parameters
ifold = 0
xselector = np.ones(mdata)
cMMR.split_train_test(xselector, ifold)
best_param = np.array(
[np.mean(xbest_param[:, :, i]) for i in range(nparam)])
cMMR.penalty.c = best_param[0]
cMMR.penalty.d = best_param[1]
cMMR.XKernel[0].kernel_params.ipar1 = best_param[2]
cMMR.XKernel[0].kernel_params.ipar2 = best_param[3]
cMMR.compute_kernels()
cMMR.Y0 = cMMR.YKernel.get_train(cMMR.itrain) ## candidates
## training with the best parameters
print('Full training')
cOptDual = cMMR.mmr_train()
np.savetxt(cdata_store.sbasedir+cdata_store.dual_params,cMMR.dual.alpha, \
fmt='%9.4f')
np.savetxt(cdata_store.sbasedir+cdata_store.kernel_params,best_param[2:], \
fmt='%9.4f')
print(xclassconfusion)
print('Bye')
return
def mmr_main(iworkmode):
params = mmr_setparams.cls_params()
## @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
list_features=["annot","DenseHue","DenseHueV3H1", \
"DenseSift","DenseSiftV3H1","Gist", \
"HarrisHue","HarrisHueV3H1","HarrisSift", \
"HarrisSiftV3H1","Hsv","HsvV3H1","Lab", \
"LabV3H1","Rgb","RgbV3H1"]
## data files in the corresponding directories
datadirs = ['corel5k', 'espgame', 'iaprtc12', 'mirflickr', 'pascal07']
## Example lfeatures=[4,8] means we selected the features:
## "DenseSiftV3H1" and "HarrisSift"
lfeatures = [4]
params.ninputview = len(lfeatures)
idata = 0 ## process corel5k data set
xdatacls = mmr_mmr_cls.cls_mmr(params.ninputview)
nfold = xdatacls.nfold
nrepeat = xdatacls.nrepeat
print('Xbias:', xdatacls.xbias)
cdata_store = mmr_load_data.cls_data_load()
cdata_store.load_data(xdatacls, idata, lfeatures)
mdata = xdatacls.mdata
## initializing the array collecting the results
nscore = 4
nipar = 1
if xdatacls.crossval_mode == 0: ## random
nfold0 = nfold
xresult_test = np.zeros((nipar, nrepeat, nfold0))
xresult_train = np.zeros((nipar, nrepeat, nfold0))
xpr = np.zeros((nipar, nrepeat, nfold0, nscore))
elif xdatacls.crossval_mode == 1: ## predefined trianing and test
nrepeat = 1
nfold0 = 1
xresult_test = np.zeros((nipar, nrepeat, nfold0))
xresult_train = np.zeros((nipar, nrepeat, nfold0))
xpr = np.zeros((nipar, nrepeat, nfold0, nscore))
## -----------------------------------------------
print('Output kernel type: ', xdatacls.YKernel.kernel_params.kernel_type)
for i in range(params.ninputview):
print(i, 'Input kernel type: ',
xdatacls.XKernel[i].kernel_params.kernel_type)
## -------------------------------------------------
xcross = np.zeros((mdata, mdata))
xtime = np.zeros(5)
## ############################################################
nparam = 4 ## C,D,par1,par2
xbest_param = np.zeros((nrepeat, nfold0, nparam))
for iipar in range(nipar):
print('===================================================')
for irepeat in range(nrepeat):
xdatacls.prepare_repetition_training(nfold0)
for ifold in range(nfold0):
xdatacls.prepare_fold_training(ifold)
## validation to choose the best parameters
print('Validation')
t0 = time.clock()
xdatacls.set_validation()
cvalidation = mmr_validation_cls.cls_mmr_validation()
cvalidation.validation_rkernel = xdatacls.XKernel[0].title
best_param = cvalidation.mmr_validation(xdatacls)
xtime[0] = time.clock() - t0
print('Best parameters found by validation')
print('c: ', best_param.c)
print('d: ', best_param.d)
print('par1: ', best_param.par1)
print('par2: ', best_param.par2)
xbest_param[irepeat, ifold, 0] = best_param.c
xbest_param[irepeat, ifold, 1] = best_param.d
xbest_param[irepeat, ifold, 2] = best_param.par1
xbest_param[irepeat, ifold, 3] = best_param.par2
xdatacls.compute_kernels()
xdatacls.Y0 = xdatacls.YKernel.get_train(
xdatacls.itrain) ## candidates
## training with the best parameters
print('Training')
print(xdatacls.YKernel.kernel_params.kernel_type, \
xdatacls.YKernel.kernel_params.ipar1, \
xdatacls.YKernel.kernel_params.ipar2)
for iview in range(xdatacls.ninputview):
print(xdatacls.XKernel[iview].kernel_params.kernel_type, \
xdatacls.XKernel[iview].kernel_params.ipar1, \
xdatacls.XKernel[iview].kernel_params.ipar2)
t0 = time.clock()
cOptDual = xdatacls.mmr_train()
xtime[1] = time.clock() - t0
## cls transfers the dual variables to the test procedure
## compute tests
## check the train accuracy
print('Test')
cPredictTra = xdatacls.mmr_test(cOptDual, itraindata=0)
## counts the proportion the ones predicted correctly
## ######################################
if xdatacls.itestmode == 2:
print('Test knn')
ypred=inverse_knn(xdatacls.YKernel.get_Y0(xdatacls.itrain), \
cPredictTra)
else:
ypred = cPredictTra.zPred
cEvaluationTra= \
mmr_eval_binvector(xdatacls.YKernel.get_train(xdatacls.itrain), \
ypred)
xresult_train[iipar, irepeat, ifold] = cEvaluationTra.accuracy
print('>>>>>>>>>>>\n', cEvaluationTra.confusion)
## ######################################
## check the test accuracy
t0 = time.clock()
cPredictTes = xdatacls.mmr_test(cOptDual, itraindata=1)
## counts the proportion the ones predicted correctly
if xdatacls.itestmode == 2:
ypred=inverse_knn(xdatacls.YKernel.get_Y0(xdatacls.itrain), \
cPredictTes)
else:
ypred = cPredictTes.zPred
## cEvaluationTes=mmr_eval_binvector(cData.YTest,cPredictTes.zPred)
cEvaluationTes= \
mmr_eval_binvector(xdatacls.YKernel.get_test(xdatacls.itest), \
ypred)
xtime[2] = time.clock() - t0
xresult_test[iipar, irepeat, ifold] = cEvaluationTes.accuracy
xpr[iipar, irepeat, ifold, 0] = cEvaluationTes.precision
xpr[iipar, irepeat, ifold, 1] = cEvaluationTes.recall
xpr[iipar, irepeat, ifold, 2] = cEvaluationTes.f1
xpr[iipar, irepeat, ifold, 3] = cEvaluationTes.accuracy
print(cEvaluationTes.confusion)
print(cEvaluationTes.classconfusion)
try:
xclassconfusion += cEvaluationTes.classconfusion
except:
(n, n) = cEvaluationTes.classconfusion.shape
xclassconfusion = np.zeros((n, n))
xclassconfusion += cEvaluationTes.classconfusion
## mmr_eval_label(ZW,iPre,YTesN,Y0,kit_data,itest,params)
## ####################################
print('Parameter:',iipar,'Repetition: ',irepeat, \
'Fold: ',ifold)
mmr_report.mmr_report('Result on one fold',
xresult_train[iipar,irepeat,ifold], \
xresult_test[iipar,irepeat,ifold], \
xpr[iipar,irepeat,ifold,:])
print(
np.sum(xpr[iipar, irepeat, :ifold + 1, :], 0) /
(ifold + 1))
mmr_report.mmr_report('Result on one repetition',
np.mean(xresult_train[iipar,irepeat,:]), \
np.mean(xresult_test[iipar,irepeat,:]), \
np.mean(xpr[iipar,irepeat,:,:],0))
mmr_report.mmr_report('Result on all repetitions @@@@@@@',
np.mean(xresult_train[iipar,:,:].flatten()), \
np.mean(xresult_test[iipar,:,:].flatten()), \
np.mean(np.mean(xpr[iipar,:,:,:],0),0))
print('Average best parameters')
xlabels = ('c', 'd', 'par1', 'par2')
for i in range(nparam):
print(xlabels[i],': ',np.mean(xbest_param[:,:,i]), \
'(',np.std(xbest_param[:,:,i]),')')
print('xtime:', xtime)
sys.stdout.flush()
print('Bye')
return
def test_mvm_main(workmode):
params = mmr_setparams.cls_params()
xdatacls = mvm_mvm_cls.cls_mvm()
nfold = xdatacls.nfold
if xdatacls.itestmode == 0:
nfold0 = 1 ## active learning
else:
nfold0 = nfold ## n-fold cross validation
nparacc = 2 ## rmse, time
npar = 1
xsummary = np.zeros((npar, nparacc))
lfilenames = ["affordances_instrument_for", "affordances_patient"]
ifile = 1 ## file index in list above
lfiles = [0, 1]
lfeatures = ["PointMutualInformation", "absolute frequency"]
ifeature = 0
if xdatacls.itestmode == 3:
iloadall = 1
else:
iloadall = 0
print("lfiles:", lfilenames)
print("ifeature:", lfeatures[ifeature])
for ipar in range(npar):
## possible values
Y0 = np.array([-1, 0, 1])
ctables = webrel_load_data.cls_label_files()
print(ctables.listcsv[ifile])
(xdata, nrow2, ncol2, ifixtrain, ifixtest) = ctables.load_objobj_act(lfiles, ifeature)
xdatacls.load_data(xdata, xdatacls.categorymax, int(nrow2), int(ncol2), Y0)
xdatacls.ifixtrain = ifixtrain
xdatacls.ifixtest = ifixtest
scombine = ""
if xdatacls.itestmode == 0:
if xdatacls.ibootstrap == 0:
fname = "xresultte_rand" + scombine + ".csv"
elif xdatacls.ibootstrap == 1:
fname = "xresultte_active" + scombine + ".csv"
elif xdatacls.ibootstrap == 2:
fname = "xresultte_greedy" + scombine + ".csv"
elif xdatacls.ibootstrap == 3:
fname = "xresultte_act_rand" + scombine + ".csv"
else:
fname = "xresultte_ncross" + scombine + ".csv"
xdatacls.YKernel.ymax = 10
# it will be recomputed in mvm_ranges
xdatacls.YKernel.ymin = -10
xdatacls.YKernel.yrange = 200 # it will be recomputed in classcol_ranges
xdatacls.YKernel.ystep = (xdatacls.YKernel.ymax - xdatacls.YKernel.ymin) / xdatacls.YKernel.yrange
## set_printoptions(precision=4)
nparam = 4 # C,D,par1,par2
nreport = 4 ## accuracy, precision, recall, f1
xdatacls.prepare_repetition_folding(init_train_size=100)
nrepeat0 = xdatacls.nrepeat0
nfold0 = xdatacls.nfold0
if xdatacls.itestmode == 3:
nfold0 = 1
creport = mmr_report_cls.cls_mmr_report()
creport.create_xaprf(nrepeat=nrepeat0, nfold=nfold0, nreport=nreport)
xbest_param = np.zeros((nrepeat0, nfold0, nparam))
# ############################################################
nval = max(xdatacls.YKernel.valrange) + 1
xconfusion3 = np.zeros((nrepeat0, nfold0, xdatacls.YKernel.ndim, nval, nval))
xsolvertime = 0.0
ireport = 0
for irepeat in range(nrepeat0):
xdatacls.nfold0 = xdatacls.nfold
xdatacls.prepare_repetition_training()
for ifold in range(nfold0):
xdatacls.prepare_fold_training(ifold)
# validation to choose the best parameters
print("Validation")
xdatacls.set_validation()
cvalidation = mvm_validation_cls.cls_mvm_validation()
cvalidation.validation_rkernel = xdatacls.XKernel[0].title
best_param = cvalidation.mvm_validation(xdatacls)
print("Parameters:", best_param.c, best_param.d, best_param.par1, best_param.par2)
print("Best parameters found by validation")
xbest_param[irepeat, ifold, 0] = best_param.c
xbest_param[irepeat, ifold, 1] = best_param.d
xbest_param[irepeat, ifold, 2] = best_param.par1
xbest_param[irepeat, ifold, 3] = best_param.par2
# training with the best parameters
print("training")
time0 = time.time()
cOptDual = xdatacls.mvm_train()
xsolvertime += xdatacls.solvertime
print("Training time:", time.time() - time0)
sys.stdout.flush()
# check the train accuracy
print("test on training")
# check the test accuracy
print("test on test")
time0 = time.time()
if xdatacls.ifulltest == 1:
xdatacls.xdata_tes = ctables.full_test()
xdatacls.xranges_rel_test = mvm_prepare.mvm_ranges(xdatacls.xdata_tes, xdatacls.nrow)
cPredict = xdatacls.mvm_test()
print("Test time:", time.time() - time0)
sys.stdout.flush()
filename = "predicted_missing.csv"
ctables.export_prediction(filename, xdatacls, cPredict.Zrow)
# counts the proportion the ones predicted correctly
# ####################################
time0 = time.time()
if xdatacls.knowntest == 1:
(cEval, icandidate_w, icandidate_b) = mvm_eval(
xdatacls.ieval_type, xdatacls.nrow, xdatacls, cPredict.Zrow
)
print("Evaluation time:", time.time() - time0)
(qtest, qpred, qpred0) = makearray(xdatacls, cPredict.Zrow)
if xdatacls.ieval_type in (0, 11):
creport.set_xaprf(irepeat, ifold, cEval)
elif xdatacls.ieval_type == 10:
creport.set_xaprf(irepeat, ifold, cEval)
xconfusion3[irepeat, ifold] = cEval.xconfusion3
else:
creport.set_xaprf(irepeat, ifold, cEval)
## xdatacls.icandidate_w=xdatacls.itest[icandidate_w]
## xdatacls.icandidate_b=xdatacls.itest[icandidate_b]
ireport += 1
## print(cEval.xconfusion)
if xdatacls.ieval_type in (0, 11):
for xconfrow in cEval.xconfusion:
for ditem in xconfrow:
print("%7.0f" % ditem, end="")
print()
print()
elif xdatacls.ieval_type == 10:
for xtable in cEval.xconfusion3:
xsum = np.sum(xtable)
if xsum == 0:
xsum = 1
xtable = 100 * xtable / xsum
for xconfrow in xtable:
for ditem in xconfrow:
print("%9.4f" % ditem, end="")
print()
print()
print()
# ####################################
print("*** ipar, repeatation, fold ***")
print(ipar, irepeat, ifold)
if xdatacls.itestmode == 1: ## n-fold crossvalidation
creport.report_prf(
xmask=[irepeat, ifold],
stitle="Result in one fold and one repetation",
ssubtitle="Accuracy on test",
)
if xdatacls.knowntest == 1:
creport.report_prf(
xmask=[irepeat, None],
stitle="Result in one repetation",
ssubtitle="Mean and std of the accuracy on test",
)
sys.stdout.flush()
if xdatacls.knowntest == 1:
(xmean, xstd) = creport.report_prf(
xmask=[None, None],
stitle="***** Overall result ****",
ssubtitle="Mean and std of the accuracy on test + error",
)
xsummary[ipar, 0] = xmean[0]
xsummary[ipar, 1] = xsolvertime / (nrepeat0 * nfold0)
if xdatacls.itestmode == 3:
filename = "predicted_missing.csv"
## ctables.export_prediction(filename,xdatacls,cPredict.Zrow)
## (qtest,qpred,qpred0)=makearray(xdatacls,cPredict.Zrow)
print("Average best parameters")
xlabels = ("c", "d", "par1", "par2")
for i in range(nparam):
print(xlabels[i], ": ", np.mean(xbest_param[:, :, i]), "(", np.std(xbest_param[:, :, i]), ")")
if xdatacls.knowntest == 1:
print("$$$$$$$$$ Summary results:")
(m, n) = xsummary.shape
for i in range(m):
for j in range(n):
print("%10.4f" % xsummary[i, j], end="")
print()
## np.savetxt(fname,xresultte[:ireport,0,:],delimiter=',',fmt='%6.4f')
print("Bye")
return
def test_mvm_main(workmode):
params=mmr_setparams.cls_params()
xdatacls=mvm_mvm_cls.cls_mvm()
nfold=xdatacls.nfold
if xdatacls.itestmode==0:
nfold0=1 ## active learning
else:
nfold0=nfold ## n-fold cross validation
nparacc=2 ## rmse, time
npar=1
xsummary=np.zeros((npar,nparacc))
## ['full','full_20','full_40','full_60', \
## 'known','known_20','known_40','known_60']
ifile1=0 ## file index in list known
ifile2=0 ## file index in list full
iknown1=1 ## known
iknown2=0 ## full
iloadall=1 ## =0 one file for crossvalidation =1 two files: training + test
print('iknown1:',iknown1,'iknown2:',iknown2)
print('ifile1:',ifile1,'ifile2:',ifile2)
for ipar in range(npar):
## possible values
Y0=np.array([0,1])
ctables=kingsc_load_data.cls_label_files() ## data loading object
print(ctables.listknown[ifile1])
print(ctables.listfull[ifile2])
if iloadall==0: ## only one file is loaded for cross validation
(xdata,nrow2,ncol2)=ctables.load_onefile(iknown1,ifile1)
xdatacls.load_data(xdata,xdatacls.categorymax, \
int(nrow2),int(ncol2),Y0)
else: ## the first file gives trining the second serves as test
(xdata,nrow2,ncol2,ifixtrain,ifixtest)=ctables.load_twofiles( \
iknown1,iknown2,ifile1,ifile2)
xdatacls.load_data(xdata,xdatacls.categorymax, \
int(nrow2),int(ncol2),Y0)
xdatacls.ifixtrain=ifixtrain
xdatacls.ifixtest=ifixtest
scombine=''
if xdatacls.itestmode==0:
if xdatacls.ibootstrap==0:
fname='xresultte_rand'+scombine+'.csv'
elif xdatacls.ibootstrap==1:
fname='xresultte_active'+scombine+'.csv'
elif xdatacls.ibootstrap==2:
fname='xresultte_greedy'+scombine+'.csv'
elif xdatacls.ibootstrap==3:
fname='xresultte_act_rand'+scombine+'.csv'
else:
fname='xresultte_ncross'+scombine+'.csv'
xdatacls.YKernel.ymax=1
# it will be recomputed in mvm_ranges
xdatacls.YKernel.ymin=0
xdatacls.YKernel.yrange=100 # it will be recomputed in classcol_ranges
xdatacls.YKernel.ystep=(xdatacls.YKernel.ymax-xdatacls.YKernel.ymin) \
/xdatacls.YKernel.yrange
## set_printoptions(precision=4)
nparam=4 # C,D,par1,par2
nreport=4 ## accuracy, precision, recall, f1
xdatacls.prepare_repetition_folding(init_train_size=100)
nrepeat0=xdatacls.nrepeat0
nfold0=xdatacls.nfold0
creport=mmr_report_cls.cls_mmr_report()
creport.create_xaprf(nrepeat=nrepeat0,nfold=nfold,nreport=nreport)
xbest_param=np.zeros((nrepeat0,nfold0,nparam))
# ############################################################
nval=max(xdatacls.YKernel.valrange)+1
xconfusion3=np.zeros((nrepeat0,nfold0,xdatacls.YKernel.ndim,nval,nval))
xsolvertime=0.0
ireport=0
for irepeat in range(nrepeat0):
xdatacls.nfold0=xdatacls.nfold
xdatacls.prepare_repetition_training()
## nfold0=1
for ifold in range(nfold0):
xdatacls.prepare_fold_training(ifold)
# validation to choose the best parameters
print('Validation')
xdatacls.set_validation()
cvalidation=mvm_validation_cls.cls_mvm_validation()
cvalidation.validation_rkernel=xdatacls.XKernel[0].title
best_param=cvalidation.mvm_validation(xdatacls)
print('Parameters:',best_param.c,best_param.d, \
best_param.par1,best_param.par2)
print('Best parameters found by validation')
xbest_param[irepeat,ifold,0]=best_param.c
xbest_param[irepeat,ifold,1]=best_param.d
xbest_param[irepeat,ifold,2]=best_param.par1
xbest_param[irepeat,ifold,3]=best_param.par2
# training with the best parameters
print('training')
time0=time.time()
cOptDual= xdatacls.mvm_train()
xsolvertime+=xdatacls.solvertime
print('Training time:',time.time()-time0)
sys.stdout.flush()
# check the train accuracy
print('test on training')
# check the test accuracy
print('test on test')
time0=time.time()
# xdatacls.xdata_tes=ctables.full_test()
# xdatacls.xranges_rel_test=mvm_prepare.mvm_ranges(xdatacls.xdata_tes, \
# xdatacls.nrow)
cPredict=xdatacls.mvm_test()
print('Test time:',time.time()-time0)
sys.stdout.flush()
## ctables.export_prediction(cPredict.Zrow)
# counts the proportion the ones predicted correctly
# ####################################
time0=time.time()
if xdatacls.knowntest==1:
(cEval,icandidate_w,icandidate_b)=mvm_eval(xdatacls.ieval_type, \
xdatacls.nrow, \
xdatacls,cPredict.Zrow)
print('Evaluation time:',time.time()-time0)
## (qtest,qpred,qpred0)=makearray(xdatacls,cPredict.Zrow)
if xdatacls.ieval_type in (0,11):
creport.set_xaprf(irepeat,ifold,cEval)
elif xdatacls.ieval_type==10:
creport.set_xaprf(irepeat,ifold,cEval)
xconfusion3[irepeat,ifold]=cEval.xconfusion3
else:
creport.set_xaprf(irepeat,ifold,cEval)
## xdatacls.icandidate_w=xdatacls.itest[icandidate_w]
## xdatacls.icandidate_b=xdatacls.itest[icandidate_b]
ireport+=1
## print(cEval.xconfusion)
if xdatacls.ieval_type in (0,11):
for xconfrow in cEval.xconfusion:
for ditem in xconfrow:
print('%7.0f'%ditem,end='')
print()
print()
elif xdatacls.ieval_type==10:
for xtable in cEval.xconfusion3:
xsum=np.sum(xtable)
if xsum==0:
xsum=1
xtable=100*xtable/xsum
for xconfrow in xtable:
for ditem in xconfrow:
print('%9.4f'%ditem,end='')
print()
print()
print()
# ####################################
print('*** ipar, repeatation, fold ***')
print(ipar,irepeat,ifold)
if xdatacls.itestmode==1: ## n-fold crossvalidation
creport.report_prf(xmask=[irepeat,ifold], \
stitle='Result in one fold and one repetation', \
ssubtitle='Accuracy on test')
if xdatacls.knowntest==1:
creport.report_prf(xmask=[irepeat,None], \
stitle='Result in one repetation', \
ssubtitle='Mean and std of the accuracy on test')
sys.stdout.flush()
if xdatacls.knowntest==1:
(xmean,xstd)=creport.report_prf(xmask=[None,None], \
stitle='***** Overall result ****', \
ssubtitle='Mean and std of the accuracy on test + error')
xsummary[ipar,0]=xmean[0]
xsummary[ipar,1]=xsolvertime/(nrepeat0*nfold0)
if iloadall==1:
filename='predicted_missing'
if iknown1==1:
filename+='_'+ctables.listknown[ifile1]
else:
filename+='_'+ctables.listfull[ifile1]
if iknown2==1:
filename+='_'+ctables.listknown[ifile2]
else:
filename+='_'+ctables.listfull[ifile2]
filename+='.csv'
ctables.export_test_prediction(filename,xdatacls,cPredict.Zrow)
## (qtest,qpred,qpred0)=makearray(xdatacls,cPredict.Zrow)
print('Average best parameters')
xlabels=('c','d','par1','par2')
for i in range(nparam):
print(xlabels[i],': ',np.mean(xbest_param[:,:,i]), \
'(',np.std(xbest_param[:,:,i]),')')
if xdatacls.knowntest==1:
print('$$$$$$$$$ Summary results:')
(m,n)=xsummary.shape
for i in range(m):
for j in range(n):
print('%10.4f'%xsummary[i,j],end='')
print()
## np.savetxt(fname,xresultte[:ireport,0,:],delimiter=',',fmt='%6.4f')
print('Bye')
return
