outputs=model.outputs(),
on_unused_input='ignore',
givens=[(todropout, np.float32(0.0))])
#use_model = theano.function(inputs=[l0.data], outputs=l1_7sm.output, on_unused_input='ignore', givens=[(todropout,np.float32(0.0))], no_default_updates=True)
print "COMPILE3"
vis_binary_model = theano.function(inputs=[l0.data, l0.label, a, b],
outputs=model.outputs() + [obinary],
on_unused_input='ignore',
givens=[(todropout, np.float32(0.0))],
no_default_updates=True)
valid_binary_model = theano.function(inputs=[l0.data, l0.label, a, b],
outputs=[_tp, _fp, _tn, _fn],
on_unused_input='ignore',
givens=[(todropout, np.float32(0.0))],
no_default_updates=True)
with safefile('modelf789_2') as loadf:
if loadf:
model.load(loadf.rb())
if len(sys.argv) > 1 and sys.argv[1] == 'test':
print "Get precision & recall"
def getF(ab):
pr = []
#valid = inp[230:330]
#voutp = oup[230:330]
a, b = ab.astype('f')
for j in range(100 / BS):
#sys.stdout.write('.')
#sys.stdout.flush()
v = valid_binary_model(valid[j * BS:(j + 1) * BS],
def trainroutine(ftrain,model,savename,vispath,fdatagen,fvis=None,fcheck=None,fcheckgen=None,TOLTIMES=5,BATCHSTEP=10,LEARNRATEVAR=None,LEARNRATETARGET=10.0,LEARNADJUST=1.01, remotemonitor = False, sameranks = [], longrangecheck = None, longrangeperiod = None,totalsteps = None):
global TRAINSETTINGS
TRAINSETTINGS.TOLTIMES = TOLTIMES
TRAINSETTINGS.BATCHSTEP = BATCHSTEP
TRAINSETTINGS.LEARNRATEVAR = LEARNRATEVAR
TRAINSETTINGS.LEARNRATETARGET = LEARNRATETARGET
TRAINSETTINGS.LEARNADJUST = LEARNADJUST
TRAINSETTINGS.TOTALSTEPS = totalsteps
from layerbase import safefile
import sys, os
from fractallayer import dtypeX
if remotemonitor!=False:
import modelrecord
if remotemonitor==None: modrec = remotemonitor.Record()
elif isinstance(remotemonitor, tuple): modrec = modelrecord.Record(*remotemonitor)
else: modrec = modelrecord.Record(remotemonitor)
modrec.genmeta(model, sameranks)
else:
modrec = None
with safefile(savename) as loadf:
if loadf:
model.load(loadf.rb())
LOSS0 = 1e100
tol = 0
l = d = 0
if vispath!=None:
if not os.path.exists(vispath):
os.mkdir(vispath)
MPDrawInitializer(vispath)
if isinstance(fdatagen, (str,tuple,list)):
fdatagen = MPTwoAheadProducer(fdatagen)
else:
fdatagen = TwoAheadProducer(fdatagen)
step = 0
lrstep = 0
if longrangecheck!=None and longrangeperiod==None:
longrangeperiod = BATCHSTEP
if longrangeperiod!=None:
TRAINSETTINGS.LONGRANGEPERIOD = longrangeperiod
while True:
step += 1
if TRAINSETTINGS.TOTALSTEPS!=None and step>TRAINSETTINGS.TOTALSTEPS: break
while True:
try:
gen = fdatagen()
loss,upd = [float(t) for t in ftrain(*gen)]
break
except KeyboardInterrupt:raise
except SystemExit:raise
except:
import traceback
traceback.print_exc()
sys.stdout.write('*')
continue
l += loss
d += upd
sys.stdout.write('.')
sys.stdout.flush()
if step % TRAINSETTINGS.BATCHSTEP == TRAINSETTINGS.BATCHSTEP-1:
print d,l,
if TRAINSETTINGS.LEARNRATEVAR!=None:
lval = TRAINSETTINGS.LEARNRATEVAR.get_value()
if d>TRAINSETTINGS.LEARNRATETARGET*TRAINSETTINGS.BATCHSTEP: lval /= TRAINSETTINGS.LEARNADJUST
else: lval *= TRAINSETTINGS.LEARNADJUST
TRAINSETTINGS.LEARNRATEVAR.set_value(dtypeX(lval))
print lval,
if modrec!=None:
modrec.R()
modrec.Rlt(l)
modrec.Rd()
l = d = 0
if vispath!=None:
print "DRAW"
#Draw model
drawlayers = []
layer = 0
for i in model.paramlayers():
if len(i.params)<1: continue
if len(i.params)>2:
if hasattr(i,'reshape'):
reshape = i.reshape
else:
reshape = [None]*len(i.params)
for j,rj in zip(i.params,reshape):
s = j.get_value()
vsh = [i for i in s.shape if i>1]
if len(vsh)<2: continue
layer += 1
drawlayers.append((layer, s, rj))
else:
layer += 1
drawlayers.append((layer, i.params[0].get_value(), i.reshape if hasattr(i,'reshape') and i.reshape!=None else None))
resplayers = fvis(*gen) if fvis!=None else []
MPDrawWriter(drawlayers,resplayers)
else:
print
#Check validset
if fcheckgen!=None and fcheck!=None:
LOSS1 = 0.0
for j in fcheckgen():
sys.stdout.write('.')
sys.stdout.flush()
LOSS1 += fcheck(*j)
print LOSS1
if modrec!=None:
modrec.Rlv(LOSS1)
if LOSS1>LOSS0:
print "Converge on validset"
tol+=1
if tol>TRAINSETTINGS.TOLTIMES:
sys.exit(0)
else:
tol=0
print "NEW LOSS",LOSS1
LOSS0 = LOSS1
if longrangecheck!=None:
lrstep += 1
if lrstep%TRAINSETTINGS.LONGRANGEPERIOD == TRAINSETTINGS.LONGRANGEPERIOD-1:
try:
result = longrangecheck()
modrec.Rfloat(result)
except KeyboardInterrupt: raise
except SystemExit: raise
except:
import traceback
traceback.print_exc()
#Commit
if modrec!=None:
modrec.C()
#Save model
with safefile(savename) as savef:
model.save(savef.wb())
print "COMPILE0"
train_model = theano.function(inputs=[l0.data, l0.label],
outputs=[cost,updating], updates=updates,
givens=[(todropout,np.float32(0.0))])
print "COMPILE1"
valid_model = theano.function(inputs=[l0.data, l0.label], outputs=cost, on_unused_input='ignore', givens=[(todropout,np.float32(0.0))], no_default_updates=True)
print "COMPILE2"
vis_model = theano.function(inputs=[l0.data, l0.label],
outputs=model.outputs(),
on_unused_input='ignore',
givens=[(todropout,np.float32(0.0))])
use_model = theano.function(inputs=[l0.data], outputs=l1_7sm.output, on_unused_input='ignore', givens=[(todropout,np.float32(0.0))], no_default_updates=True)
print "COMPILE3"
vis_binary_model = theano.function(inputs=[l0.data, l0.label], outputs=model.outputs() + [obinary], on_unused_input='ignore', givens=[(todropout,np.float32(0.0))], no_default_updates=True)
valid_binary_model = theano.function(inputs=[l0.data, l0.label], outputs=[_tp,_fp,_tn,_fn], on_unused_input='ignore', givens=[(todropout,np.float32(0.0))], no_default_updates=True)
with safefile('modelfallface2') as loadf:
if loadf:
model.load(loadf.rb())
if len(sys.argv)>1 and sys.argv[1] == 'test':
print "Get precision & recall"
pr = []
valid = inp[230:330]
voutp = oup[230:330]
for j in range(100/BS):
#sys.stdout.write('.')
#sys.stdout.flush()
v=valid_binary_model(valid[j*BS:(j+1)*BS],voutp[j*BS:(j+1)*BS])
resp = vis_binary_model(valid[j*BS:(j+1)*BS],voutp[j*BS:(j+1)*BS])
layer = 0
for i in resp:
def trainroutine(ftrain,
model,
savename,
vispath,
fdatagen,
fvis=None,
fcheck=None,
fcheckgen=None,
TOLTIMES=5,
BATCHSTEP=10,
LEARNRATEVAR=None,
LEARNRATETARGET=10.0,
LEARNADJUST=1.01,
remotemonitor=False,
sameranks=[],
longrangecheck=None,
longrangeperiod=None,
totalsteps=None):
global TRAINSETTINGS
TRAINSETTINGS.TOLTIMES = TOLTIMES
TRAINSETTINGS.BATCHSTEP = BATCHSTEP
TRAINSETTINGS.LEARNRATEVAR = LEARNRATEVAR
TRAINSETTINGS.LEARNRATETARGET = LEARNRATETARGET
TRAINSETTINGS.LEARNADJUST = LEARNADJUST
TRAINSETTINGS.TOTALSTEPS = totalsteps
from layerbase import safefile
import sys, os
from fractallayer import dtypeX
if remotemonitor != False:
import modelrecord
if remotemonitor == None: modrec = remotemonitor.Record()
elif isinstance(remotemonitor, tuple):
modrec = modelrecord.Record(*remotemonitor)
else:
modrec = modelrecord.Record(remotemonitor)
modrec.genmeta(model, sameranks)
else:
modrec = None
with safefile(savename) as loadf:
if loadf:
model.load(loadf.rb())
LOSS0 = 1e100
tol = 0
l = d = 0
if vispath != None:
if not os.path.exists(vispath):
os.mkdir(vispath)
MPDrawInitializer(vispath)
if isinstance(fdatagen, (str, tuple, list)):
fdatagen = MPTwoAheadProducer(fdatagen)
else:
fdatagen = TwoAheadProducer(fdatagen)
step = 0
lrstep = 0
if longrangecheck != None and longrangeperiod == None:
longrangeperiod = BATCHSTEP
if longrangeperiod != None:
TRAINSETTINGS.LONGRANGEPERIOD = longrangeperiod
while True:
step += 1
if TRAINSETTINGS.TOTALSTEPS != None and step > TRAINSETTINGS.TOTALSTEPS:
break
while True:
try:
gen = fdatagen()
loss, upd = [float(t) for t in ftrain(*gen)]
break
except KeyboardInterrupt:
raise
except SystemExit:
raise
except:
import traceback
traceback.print_exc()
sys.stdout.write('*')
continue
l += loss
d += upd
sys.stdout.write('.')
sys.stdout.flush()
if step % TRAINSETTINGS.BATCHSTEP == TRAINSETTINGS.BATCHSTEP - 1:
print d, l,
if TRAINSETTINGS.LEARNRATEVAR != None:
lval = TRAINSETTINGS.LEARNRATEVAR.get_value()
if d > TRAINSETTINGS.LEARNRATETARGET * TRAINSETTINGS.BATCHSTEP:
lval /= TRAINSETTINGS.LEARNADJUST
else:
lval *= TRAINSETTINGS.LEARNADJUST
TRAINSETTINGS.LEARNRATEVAR.set_value(dtypeX(lval))
print lval,
if modrec != None:
modrec.R()
modrec.Rlt(l)
modrec.Rd()
l = d = 0
if vispath != None:
print "DRAW"
#Draw model
drawlayers = []
layer = 0
for i in model.paramlayers():
if len(i.params) < 1: continue
if len(i.params) > 2:
if hasattr(i, 'reshape'):
reshape = i.reshape
else:
reshape = [None] * len(i.params)
for j, rj in zip(i.params, reshape):
s = j.get_value()
vsh = [i for i in s.shape if i > 1]
if len(vsh) < 2: continue
layer += 1
drawlayers.append((layer, s, rj))
else:
layer += 1
drawlayers.append((layer, i.params[0].get_value(),
i.reshape if hasattr(i, 'reshape')
and i.reshape != None else None))
resplayers = fvis(*gen) if fvis != None else []
MPDrawWriter(drawlayers, resplayers)
else:
print
#Check validset
if fcheckgen != None and fcheck != None:
LOSS1 = 0.0
for j in fcheckgen():
sys.stdout.write('.')
sys.stdout.flush()
LOSS1 += fcheck(*j)
print LOSS1
if modrec != None:
modrec.Rlv(LOSS1)
if LOSS1 > LOSS0:
print "Converge on validset"
tol += 1
if tol > TRAINSETTINGS.TOLTIMES:
sys.exit(0)
else:
tol = 0
print "NEW LOSS", LOSS1
LOSS0 = LOSS1
if longrangecheck != None:
lrstep += 1
if lrstep % TRAINSETTINGS.LONGRANGEPERIOD == TRAINSETTINGS.LONGRANGEPERIOD - 1:
try:
result = longrangecheck()
modrec.Rfloat(result)
except KeyboardInterrupt:
raise
except SystemExit:
raise
except:
import traceback
traceback.print_exc()
#Commit
if modrec != None:
modrec.C()
#Save model
with safefile(savename) as savef:
model.save(savef.wb())