def elastic_net(X,Y,w0,Xteste,Yteste,Xdev,Ydev,alpha_en): import matplotlib.pyplot as plt w_new=np.zeros(w0.shape) zeros=np.zeros(w0.shape) final=[0,0] sigma=1 sigma2=0.1 max_iter =40000 yyy=[] xxx=[] gmedia=[] gmediatest=[] w0=w0.todense() softt=it.softt getU=it.getU get_step=it.get_step get_func_elastic=it.get_func_elastic GetGradL=it.GetGradL lambs=[10**x for x in xrange(0,10)] xg=map(lambda x: np.log10(x),lambs) #lambs=[100] #lambs=[1000*X.shape[0]] #xg=[7] for lamb in lambs: print "A iniciar Iteracoes para lambda=",lamb w_old_old=w0 w_old=w0 yy=[] xx=[] i=0 while i<max_iter: #print "iteracao",i error=(X*w_old)-Y if i==0: step_size=0.0000001 else: #pdb.set_trace() error_old=error=(X*w_old)-Y error_old_old=(X*w_old_old)-Y alpha=get_step(w_old,w_old_old,X,error_old,error_old_old,lamb) if alpha==0: #print "**ERRO**" #print "aplha=0, impossivel continuar o algorimto" break step_size=sigma/alpha error=(X*w_old)-Y grad1=GetGradL(error,X) U=getU(w_old,step_size,grad1) K=1+step_size*lamb*(1-alpha_en) w_new=softt(U,step_size*lamb*alpha_en/K,zeros) dif=w_new-w_old dif=dif.transpose()*dif error=(X*w_new)-Y y_new=get_func_elastic(error,w_new,lamb,alpha_en) #funcao de erro #print i,"->",y_new count=0 if i!=0: while y_new>=y_old-sigma2*alpha*dif[0,0] and i<max_iter: #print "A diminuir step:",i step_size=step_size/2 U=getU(w_old,step_size,grad1) K=1+step_size*lamb*(1-alpha_en) w_new=softt(U,step_size*lamb*alpha_en/K,zeros) error=(X*w_new)-Y dif=w_new-w_old dif=dif.transpose()*dif y_new=get_func_elastic(error,w_new,lamb,alpha_en) #funcao de erro count=count+1 i=i+1 if count==10000: break if count ==10000: #print "****A SAIR****\nProvavelmente o sparsa chegou ao minimo antes de terminar o numero de iteracoes" break i=i+1 y_old=y_new w_old_old=w_old w_old = w_new yy.append(y_new) xx.append(i) media=RRegression_beta.erro(Xdev,Ydev,w_new) gmedia.append(media) gmediatest.append(RRegression_beta.erro(Xteste,Yteste,w_new)) if final[0]>media or final[0]==0: final[0]=media final[1]=lamb graphFinal=deepcopy(yy) wfinal=w_new yfinal=y_new finalxx=deepcopy(xx) zero=0.0 for J in xrange(w_new.shape[0]): if w_new[J,0]==0: zero=zero+1.0 sp=(zero/w_new.shape[0])*100 #print "percentagem:",sp yyy.append(sp) xxx.append(lamb) ''' plt.figure(1) plt.subplot(221) plt.title("Funcao de custo") plt.plot(finalxx,graphFinal,"r") plt.subplot(222) plt.title("Percentagem de W com valor =0") import pylab #print yyy #pylab.ylim([0,100]) plt.plot(xg,yyy,"b",xg,yyy,"ro") plt.subplot(223) plt.title("Evolucao do erro DEV ao longo dos lambdas") plt.plot(xg,gmedia,"b",xg,gmedia,"ro") plt.subplot(224) plt.title("Evolucao do erro teste ao longo dos lambdas") plt.plot(xg,gmediatest,"b",xg,gmediatest,"ro") plt.tight_layout() #pylab.savefig("elastic_beta.png") plt.show() ''' return wfinal,yfinal,final[1]
devsize=int(Xtotal.shape[0]*0.1)+trainsize Xtotal=Xtotal.tocsc() #Xtotal,indices=Priberam_Ridge.repara(Xtotal,indices) #train_index=xrange(trainsize) #dev_index=xrange(trainsize,devsize) #test_index=xrange(devsize,Xtotal.shape[0]) #Xtrain,Ytrain,Xtest,Ytest,Xdev,Ydev=Priberam_Ridge.separaXY(Xtotal,Ytotal) vec=sparse.csr_matrix([0 for i in xrange(Xtotal.shape[1])]) vec=vec.transpose() Ytotal=Ytotal.tocsc() W,F,lamb=Lasso.lasso(Xtotal[:trainsize,:],Ytotal[0:trainsize,0],vec,Xtotal[devsize:,:],Ytotal[devsize:,0],Xtotal[trainsize:devsize,:],Ytotal[trainsize:devsize,0],False,False) #W,F,lamb=Lasso.lasso(Xtrain,Ytrain.transpose(),vec,Xdev,Ydev.transpose(),Xtest,Ytest.transpose()) #print "ERRO:",Rfechado.erro(Xtest,Ytest,W) print "ERRO:", Rfechado.erro(Xtotal[trainsize:devsize,:],Ytotal[trainsize:devsize,:],W) ''' for k in indices: if W[indices[k]]==0: print k ''' print "PIORES 10" for coiso in sorted(np.array(W))[:10]: i=0 while W[i,0] != coiso[0]: i+=1 else: for cenas in indices: if indices[cenas]==i: print cenas , coiso, "->",i break
vec=vec.transpose() ''' W,F=elastic_net(X,Y,vec,Xteste,Yteste,Xdev,Ydev,alpha_en) print "----------erro---------" print "TESTE",RRegression_beta.erro(f+"train.txt",W,total) print "-----------------------" ''' print "-----------------------------" #W,F,lamb=elastic_net(X,Y,vec,Xteste,Yteste,Xdev,Ydev,0.0) #print "ALPHA", alpha #print "LAMBDA",lamb #print "ERRO_DEV:",Rfechado.erro(Xdev,Ydev,W) #W=elastic_net.elastic_net(Xtrain,Ytrain.transpose(),vec,Xdev,Ydev.transpose(),Xtest,Ytest.transpose(),0.5) #print "ERRO_TEST:",Rfechado.erro(Xteste,Yteste,W) #error=X*W-Y #print "OBJECTIVO:", it.get_func_elastic(error,W,1000*X.shape[0],1) #print "OBJECTIVO:", F for alpha in [0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]: print "------------------------------" W,F,lamb=elastic_net(X,Y,vec,Xteste,Yteste,Xdev,Ydev,alpha) print "ALPHA", alpha print "LAMBDA",lamb print "ERRO_DEV:",Rfechado.erro(Xdev,Ydev,W) #W=elastic_net.elastic_net(Xtrain,Ytrain.transpose(),vec,Xdev,Ydev.transpose(),Xtest,Ytest.transpose(),0.5) print "ERRO_TEST:",Rfechado.erro(Xteste,Yteste,W) #print "TRAIN",RRegression_beta.erro("../le_ficheiro/train_meta.txt",W,total) #print "-----------------------" #print "DEV",RRegression_beta.erro("../le_ficheiro/dev_meta.txt",W,total)
def lasso(X,Y,w0,Xteste,Yteste,Xdev,Ydev,max_iter,lamb): import matplotlib.pyplot as plt w_new=np.zeros(w0.shape) zeros=np.zeros(w0.shape) final=[0,0,0] sigma=1 sigma2=0.1 if max_iter==False: max_iter=6000 yyy=[] xxx=[] gmedia=[] gmediatest=[] w0=w0.todense() softt=it.softt get_step=it.get_step getU=it.getU get_func_lasso=it.get_func_lasso GetGradL=it.GetGradL #lambdas=100*X.shape[0] if lamb==False: lambs=[10**x for x in xrange(0,10)] xg=map(lambda x: np.log10(x),lambs) else: #lambs=[lamb*X.shape[0]] lambs=[lamb] xg=[2] for lamb in lambs: print "A iniciar Iteracoes para lambda=",lamb w_old_old=w0 w_old=w0 yy=[] xx=[] i=0 while i<max_iter: #pdb.set_trace() #print "iteracao",i if i==0: step_size=0.0000001 else: #pdb.set_trace() error_old=(X*w_old)-Y error_old_old=(X*w_old_old)-Y alpha=get_step(w_old,w_old_old,X,error_old,error_old_old,lamb) if alpha!=0: step_size=sigma/alpha else: i=max_iter error=(X*w_old)-Y grad1=GetGradL(error,X) U=getU(w_old,step_size,grad1) #pdb.set_trace() #w_new=[soft(x,step_size*lamb) for x in U.transpose().tolist()[0]] #w_new=sparse.csr_matrix(w_new) #w_new=w_new.transpose() w_new=softt(U,step_size*lamb,zeros) #pdb.set_trace() dif=w_new-w_old dif=dif.transpose()*dif error=(X*w_new)-Y #pdb.set_trace() y_new=get_func_lasso(error,w_new,lamb) #funcao de erro #print i,"->",y_new count=0 if i!=0: w_temp=w_new while y_new>=y_old-sigma2*alpha*dif[0,0] and i<max_iter: #while y_new>=y_old: #print "A diminuir step:",i step_size=step_size/2 U=getU(w_old,step_size,grad1) w_new=softt(U,step_size*lamb,zeros) error=(X*w_new)-Y dif=w_new-w_old dif=dif.transpose()*dif y_new=get_func_lasso(error,w_new,lamb) #funcao de custo count=count+1 i=i+1 if count==500: print "nao consegue dar um passo pequeno o sufeciente" w_new=w_old break if count ==500: #print "****A SAIR****\nProvavelmente o sparsa chegou ao minimo antes de terminar o numero de iteracoes" break #if count ==5000: #print "****A SAIR****\nProvavelmente o sparsa chegou ao minimo antes de terminar o numero de iteracoes" #break #elif i==max_iter: #pass #print "Fim das interacoes" i=i+1 y_old=y_new w_old_old=w_old w_old=w_new yy.append(y_new) xx.append(i) print "fim do algoritmo" errod=RRegression_beta.erro(Xdev,Ydev,w_new) gmedia.append(errod) errot=RRegression_beta.erro(Xteste,Yteste,w_new) gmediatest.append(errot) if final[0]>errod or final[0]==0: final[0]=errod final[1]=lamb final[2]=errot graphFinal=deepcopy(yy) wfinal=deepcopy(w_new) yfinal=deepcopy(y_new) finalxx=deepcopy(xx) zero=0.0 for J in xrange(w_new.shape[0]): if w_new[J,0]==0: zero=zero+1.0 sp=(zero/w_new.shape[0])*100 print "percentagem:",sp yyy.append(sp) xxx.append(lamb) plt.figure(1) plt.subplot(221) plt.title("Funcao de custo") import pylab pylab.ylim([0e16,7e16]) plt.plot(finalxx,graphFinal,"r",finalxx) plt.subplot(222) plt.title("Percentagem de W com valor =0") #print yyy #pylab.ylim([0,100]) plt.plot(xg,yyy,"b",xg,yyy,"ro") plt.subplot(223) plt.title("Erro DEV ao longo dos lambdas") plt.plot(xg,gmedia,"b",xg,gmedia,"ro") plt.subplot(224) plt.title("Erro teste ao longo dos lambdas") plt.plot(xg,gmediatest,"b",xg,gmediatest,"ro") #pylab.savefig("lasso_beta.png") plt.show() return wfinal,yfinal,final[1]
#Y=sparse.csr_matrix(Ytotal)[0:Y_train,0] #Yteste=sparse.csr_matrix(Ytotal)[Y_train:Y_test+Y_train,0] #Ydev=sparse.csr_matrix(Ytotal)[Y_test+Y_train:,0] vec=sparse.csr_matrix([0.0 for i in xrange(X.shape[1])]) vec=vec.transpose() #ficheiro=open(".pkl","rb") #vec=pickle.load(ficheiro) W,F,lambd=lasso(X,Y,vec,Xteste,Yteste,Xdev,Ydev,False,False) print "OBJECTIVO:",F error=X*W-Y #print "OBJECTIVO",F #print "OBJECTIVO_CALC:", it.get_func_lasso(error,W,lambd) print "----------erro---------" print "LAMBDA:",lambd print "TESTE",RRegression_beta.erro(Xteste,Yteste,W) print "-----------------------" print "PIORES 10" for coiso in sorted(np.array(W))[:10]: i=0 while W[i,0] != coiso: i+=1 else: for cenas in total: if total[cenas]==i: print cenas , coiso, "->",i break print "TOP 10" for coiso in sorted(np.array(W))[W.shape[1]-11:]: i=0
trainsize=int(Xtotal.shape[0]*0.8) devsize=int(Xtotal.shape[0]*0.1)+trainsize #Xtrain,Ytrain,Xtest,Ytest,Xdev,Ydev=PB.separaXY(Xtotal,Ytotal) #train_index=xrange(trainsize) #dev_index=xrange(trainsize,devsize) #test_index=xrange(devsize,Xtotal.shape[0]) Xtotal=Xtotal.tocsc() Ytotal=Ytotal.tocsc() Xtotal,indices=cria_dados.delstopword(Xtotal,indices,False) #Xtotal,indices=PB.tira_meta(Xtotal,indices) #vec=sparse.csr_matrix([0 for i in xrange(Xtrain.shape[1])]) vec=sparse.csr_matrix([0 for i in xrange(Xtotal[:trainsize,:].shape[1])]) vec=vec.transpose() print "-----------------------------" for alpha in [0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]: print "------------------------------" W,F,lamb=elastic_net.elastic_net(Xtotal[:trainsize,:],Ytotal[0:trainsize,0],vec,Xtotal[devsize:,:],Ytotal[devsize:,0],Xtotal[trainsize:devsize,:],Ytotal[trainsize:devsize,0],alpha) print "ALPHA", alpha print "LAMBDA",lamb print "ERRO:",Rfechado.erro(Xtotal[trainsize:devsize,:],Ytotal[trainsize:devsize,:],W) #W=elastic_net.elastic_net(Xtrain,Ytrain.transpose(),vec,Xdev,Ydev.transpose(),Xtest,Ytest.transpose(),0.5) print "ERRO_DEV:",Rfechado.erro(Xtotal[trainsize:devsize,:],Ytotal[trainsize:devsize,:],W) #W=elastic_net.elastic_net(Xtrain,Ytrain.transpose(),vec,Xdev,Ydev.transpose(),Xtest,Ytest.transpose(),0.5) print "ERRO_TEST:",Rfechado.erro(Xtotal[trainsize:devsize,:],Ytotal[trainsize:devsize,:],W) #print "ERRO:",Rfechado.erro(Xtest,Ytest,W) #print "ERRO:",Rfechado.erro(Xtotal[trainsize:devsize,:],Ytest[trainsize:devsize,:],W)
if __name__ == '__main__': f="../le_ficheiro/someta" dictionary,total,y=cria_dados.read_output(f+"train.txt") #X,Y,mediaY,stdY,mediaX=cria_dados.criaXY(dictionary,total,y,True) X,Y=cria_dados.criaXY(dictionary,total,y,False) dictionary,temp,y=cria_dados.read_output(f+"test.txt") Xteste,Yteste=cria_dados.criaXY(dictionary,total,y,False) dictionary,temp,y=cria_dados.read_output(f+"dev.txt") Xdev,Ydev=cria_dados.criaXY(dictionary,total,y,False) mediaX=0.0 mediaY=0.0 stdY=1.0 #X,total=cria_dados.delcomun(X,total) vec=sparse.csr_matrix([0.0 for i in xrange(X.shape[1])]) vec=vec.transpose() ficheiro=open("data.pkl","rb") #vec=pickle.load(ficheiro) W,F,lambd=lasso(X,Y,vec,Xteste,Yteste,Xdev,Ydev) error=X*W-Y print "OBJECTIVO",F print "OBJECTIVO_CALC:", it.get_func_lasso(error,W,lambd) print "----------erro---------" print "LAMBDA:",lambd print "TESTE",RRegression_beta.erro(f+"test.txt",W,total,mediaY,stdY,mediaX) print "-----------------------" #print "TRAIN",RRegression_beta.erro("../le_ficheiro/train_meta.txt",W,total) #print "-----------------------" #print "DEV",RRegression_beta.erro("../le_ficheiro/dev_meta.txt",W,total)