outputs=model.outputs(), on_unused_input='ignore', givens=[(todropout, np.float32(0.0))]) #use_model = theano.function(inputs=[l0.data], outputs=l1_7sm.output, on_unused_input='ignore', givens=[(todropout,np.float32(0.0))], no_default_updates=True) print "COMPILE3" vis_binary_model = theano.function(inputs=[l0.data, l0.label, a, b], outputs=model.outputs() + [obinary], on_unused_input='ignore', givens=[(todropout, np.float32(0.0))], no_default_updates=True) valid_binary_model = theano.function(inputs=[l0.data, l0.label, a, b], outputs=[_tp, _fp, _tn, _fn], on_unused_input='ignore', givens=[(todropout, np.float32(0.0))], no_default_updates=True) with safefile('modelf789_2') as loadf: if loadf: model.load(loadf.rb()) if len(sys.argv) > 1 and sys.argv[1] == 'test': print "Get precision & recall" def getF(ab): pr = [] #valid = inp[230:330] #voutp = oup[230:330] a, b = ab.astype('f') for j in range(100 / BS): #sys.stdout.write('.') #sys.stdout.flush() v = valid_binary_model(valid[j * BS:(j + 1) * BS],
def trainroutine(ftrain,model,savename,vispath,fdatagen,fvis=None,fcheck=None,fcheckgen=None,TOLTIMES=5,BATCHSTEP=10,LEARNRATEVAR=None,LEARNRATETARGET=10.0,LEARNADJUST=1.01, remotemonitor = False, sameranks = [], longrangecheck = None, longrangeperiod = None,totalsteps = None): global TRAINSETTINGS TRAINSETTINGS.TOLTIMES = TOLTIMES TRAINSETTINGS.BATCHSTEP = BATCHSTEP TRAINSETTINGS.LEARNRATEVAR = LEARNRATEVAR TRAINSETTINGS.LEARNRATETARGET = LEARNRATETARGET TRAINSETTINGS.LEARNADJUST = LEARNADJUST TRAINSETTINGS.TOTALSTEPS = totalsteps from layerbase import safefile import sys, os from fractallayer import dtypeX if remotemonitor!=False: import modelrecord if remotemonitor==None: modrec = remotemonitor.Record() elif isinstance(remotemonitor, tuple): modrec = modelrecord.Record(*remotemonitor) else: modrec = modelrecord.Record(remotemonitor) modrec.genmeta(model, sameranks) else: modrec = None with safefile(savename) as loadf: if loadf: model.load(loadf.rb()) LOSS0 = 1e100 tol = 0 l = d = 0 if vispath!=None: if not os.path.exists(vispath): os.mkdir(vispath) MPDrawInitializer(vispath) if isinstance(fdatagen, (str,tuple,list)): fdatagen = MPTwoAheadProducer(fdatagen) else: fdatagen = TwoAheadProducer(fdatagen) step = 0 lrstep = 0 if longrangecheck!=None and longrangeperiod==None: longrangeperiod = BATCHSTEP if longrangeperiod!=None: TRAINSETTINGS.LONGRANGEPERIOD = longrangeperiod while True: step += 1 if TRAINSETTINGS.TOTALSTEPS!=None and step>TRAINSETTINGS.TOTALSTEPS: break while True: try: gen = fdatagen() loss,upd = [float(t) for t in ftrain(*gen)] break except KeyboardInterrupt:raise except SystemExit:raise except: import traceback traceback.print_exc() sys.stdout.write('*') continue l += loss d += upd sys.stdout.write('.') sys.stdout.flush() if step % TRAINSETTINGS.BATCHSTEP == TRAINSETTINGS.BATCHSTEP-1: print d,l, if TRAINSETTINGS.LEARNRATEVAR!=None: lval = TRAINSETTINGS.LEARNRATEVAR.get_value() if d>TRAINSETTINGS.LEARNRATETARGET*TRAINSETTINGS.BATCHSTEP: lval /= TRAINSETTINGS.LEARNADJUST else: lval *= TRAINSETTINGS.LEARNADJUST TRAINSETTINGS.LEARNRATEVAR.set_value(dtypeX(lval)) print lval, if modrec!=None: modrec.R() modrec.Rlt(l) modrec.Rd() l = d = 0 if vispath!=None: print "DRAW" #Draw model drawlayers = [] layer = 0 for i in model.paramlayers(): if len(i.params)<1: continue if len(i.params)>2: if hasattr(i,'reshape'): reshape = i.reshape else: reshape = [None]*len(i.params) for j,rj in zip(i.params,reshape): s = j.get_value() vsh = [i for i in s.shape if i>1] if len(vsh)<2: continue layer += 1 drawlayers.append((layer, s, rj)) else: layer += 1 drawlayers.append((layer, i.params[0].get_value(), i.reshape if hasattr(i,'reshape') and i.reshape!=None else None)) resplayers = fvis(*gen) if fvis!=None else [] MPDrawWriter(drawlayers,resplayers) else: print #Check validset if fcheckgen!=None and fcheck!=None: LOSS1 = 0.0 for j in fcheckgen(): sys.stdout.write('.') sys.stdout.flush() LOSS1 += fcheck(*j) print LOSS1 if modrec!=None: modrec.Rlv(LOSS1) if LOSS1>LOSS0: print "Converge on validset" tol+=1 if tol>TRAINSETTINGS.TOLTIMES: sys.exit(0) else: tol=0 print "NEW LOSS",LOSS1 LOSS0 = LOSS1 if longrangecheck!=None: lrstep += 1 if lrstep%TRAINSETTINGS.LONGRANGEPERIOD == TRAINSETTINGS.LONGRANGEPERIOD-1: try: result = longrangecheck() modrec.Rfloat(result) except KeyboardInterrupt: raise except SystemExit: raise except: import traceback traceback.print_exc() #Commit if modrec!=None: modrec.C() #Save model with safefile(savename) as savef: model.save(savef.wb())
print "COMPILE0" train_model = theano.function(inputs=[l0.data, l0.label], outputs=[cost,updating], updates=updates, givens=[(todropout,np.float32(0.0))]) print "COMPILE1" valid_model = theano.function(inputs=[l0.data, l0.label], outputs=cost, on_unused_input='ignore', givens=[(todropout,np.float32(0.0))], no_default_updates=True) print "COMPILE2" vis_model = theano.function(inputs=[l0.data, l0.label], outputs=model.outputs(), on_unused_input='ignore', givens=[(todropout,np.float32(0.0))]) use_model = theano.function(inputs=[l0.data], outputs=l1_7sm.output, on_unused_input='ignore', givens=[(todropout,np.float32(0.0))], no_default_updates=True) print "COMPILE3" vis_binary_model = theano.function(inputs=[l0.data, l0.label], outputs=model.outputs() + [obinary], on_unused_input='ignore', givens=[(todropout,np.float32(0.0))], no_default_updates=True) valid_binary_model = theano.function(inputs=[l0.data, l0.label], outputs=[_tp,_fp,_tn,_fn], on_unused_input='ignore', givens=[(todropout,np.float32(0.0))], no_default_updates=True) with safefile('modelfallface2') as loadf: if loadf: model.load(loadf.rb()) if len(sys.argv)>1 and sys.argv[1] == 'test': print "Get precision & recall" pr = [] valid = inp[230:330] voutp = oup[230:330] for j in range(100/BS): #sys.stdout.write('.') #sys.stdout.flush() v=valid_binary_model(valid[j*BS:(j+1)*BS],voutp[j*BS:(j+1)*BS]) resp = vis_binary_model(valid[j*BS:(j+1)*BS],voutp[j*BS:(j+1)*BS]) layer = 0 for i in resp:
def trainroutine(ftrain, model, savename, vispath, fdatagen, fvis=None, fcheck=None, fcheckgen=None, TOLTIMES=5, BATCHSTEP=10, LEARNRATEVAR=None, LEARNRATETARGET=10.0, LEARNADJUST=1.01, remotemonitor=False, sameranks=[], longrangecheck=None, longrangeperiod=None, totalsteps=None): global TRAINSETTINGS TRAINSETTINGS.TOLTIMES = TOLTIMES TRAINSETTINGS.BATCHSTEP = BATCHSTEP TRAINSETTINGS.LEARNRATEVAR = LEARNRATEVAR TRAINSETTINGS.LEARNRATETARGET = LEARNRATETARGET TRAINSETTINGS.LEARNADJUST = LEARNADJUST TRAINSETTINGS.TOTALSTEPS = totalsteps from layerbase import safefile import sys, os from fractallayer import dtypeX if remotemonitor != False: import modelrecord if remotemonitor == None: modrec = remotemonitor.Record() elif isinstance(remotemonitor, tuple): modrec = modelrecord.Record(*remotemonitor) else: modrec = modelrecord.Record(remotemonitor) modrec.genmeta(model, sameranks) else: modrec = None with safefile(savename) as loadf: if loadf: model.load(loadf.rb()) LOSS0 = 1e100 tol = 0 l = d = 0 if vispath != None: if not os.path.exists(vispath): os.mkdir(vispath) MPDrawInitializer(vispath) if isinstance(fdatagen, (str, tuple, list)): fdatagen = MPTwoAheadProducer(fdatagen) else: fdatagen = TwoAheadProducer(fdatagen) step = 0 lrstep = 0 if longrangecheck != None and longrangeperiod == None: longrangeperiod = BATCHSTEP if longrangeperiod != None: TRAINSETTINGS.LONGRANGEPERIOD = longrangeperiod while True: step += 1 if TRAINSETTINGS.TOTALSTEPS != None and step > TRAINSETTINGS.TOTALSTEPS: break while True: try: gen = fdatagen() loss, upd = [float(t) for t in ftrain(*gen)] break except KeyboardInterrupt: raise except SystemExit: raise except: import traceback traceback.print_exc() sys.stdout.write('*') continue l += loss d += upd sys.stdout.write('.') sys.stdout.flush() if step % TRAINSETTINGS.BATCHSTEP == TRAINSETTINGS.BATCHSTEP - 1: print d, l, if TRAINSETTINGS.LEARNRATEVAR != None: lval = TRAINSETTINGS.LEARNRATEVAR.get_value() if d > TRAINSETTINGS.LEARNRATETARGET * TRAINSETTINGS.BATCHSTEP: lval /= TRAINSETTINGS.LEARNADJUST else: lval *= TRAINSETTINGS.LEARNADJUST TRAINSETTINGS.LEARNRATEVAR.set_value(dtypeX(lval)) print lval, if modrec != None: modrec.R() modrec.Rlt(l) modrec.Rd() l = d = 0 if vispath != None: print "DRAW" #Draw model drawlayers = [] layer = 0 for i in model.paramlayers(): if len(i.params) < 1: continue if len(i.params) > 2: if hasattr(i, 'reshape'): reshape = i.reshape else: reshape = [None] * len(i.params) for j, rj in zip(i.params, reshape): s = j.get_value() vsh = [i for i in s.shape if i > 1] if len(vsh) < 2: continue layer += 1 drawlayers.append((layer, s, rj)) else: layer += 1 drawlayers.append((layer, i.params[0].get_value(), i.reshape if hasattr(i, 'reshape') and i.reshape != None else None)) resplayers = fvis(*gen) if fvis != None else [] MPDrawWriter(drawlayers, resplayers) else: print #Check validset if fcheckgen != None and fcheck != None: LOSS1 = 0.0 for j in fcheckgen(): sys.stdout.write('.') sys.stdout.flush() LOSS1 += fcheck(*j) print LOSS1 if modrec != None: modrec.Rlv(LOSS1) if LOSS1 > LOSS0: print "Converge on validset" tol += 1 if tol > TRAINSETTINGS.TOLTIMES: sys.exit(0) else: tol = 0 print "NEW LOSS", LOSS1 LOSS0 = LOSS1 if longrangecheck != None: lrstep += 1 if lrstep % TRAINSETTINGS.LONGRANGEPERIOD == TRAINSETTINGS.LONGRANGEPERIOD - 1: try: result = longrangecheck() modrec.Rfloat(result) except KeyboardInterrupt: raise except SystemExit: raise except: import traceback traceback.print_exc() #Commit if modrec != None: modrec.C() #Save model with safefile(savename) as savef: model.save(savef.wb())