ut.mprint("Done!")
else:
# Load the last model in pretraining
wcard = os.path.dirname(wts)+"/pretrain/iter_*.model.npz"
lst=[(l,int(re.match('.*/.*_(\d+)',l).group(1))) for l in glob(wcard)]
mfn = max(lst, key=lambda x: x[1])[0]
ut.mprint("Start with pre-trained model " + mfn )
ut.loadNet(mfn,model,sess)
#########################################################################
# Main Training loop
stop=False
ut.mprint("Starting from Iteration %d" % niter)
sess.run(tset.fetchOp,feed_dict=tset.fdict())
## Learning rate, if resume at an iteration after drop
if niter >= 4e5 and niter < 5e5:
sess.run(tf.assign(lr, LR/np.sqrt(10.0)))
elif niter == 5e5:
sess.run(tf.assign(lr, LR/10.0))
while niter < MAXITER and not ut.stop:
## Validate model every so often
if niter % VALFREQ == 0:
ut.mprint("Validating model")
val_iter = vset.ndata // BSZ
vloss, vset.niter = 0., 0
sess.run(vset.fetchOp,feed_dict=vset.fdict())
intra_op_parallelism_threads=int(nthr)))
sess.run(tf.global_variables_initializer())
# Load model
wts = np.load(mfile)
ph = tf.placeholder(tf.float32)
for k in model.weights.keys():
wvar = model.weights[k]
wk = wts[k].reshape(wvar.get_shape())
sess.run(wvar.assign(ph), feed_dict={ph: wk})
# Get population stat
bnnms, bnwts = list(model.bnwts.keys()), list(model.bnwts.values())
bns = {k: [] for k in bnnms}
for i in range(niter):
sess.run(tset.fetchOp, feed_dict=tset.fdict())
tbns = sess.run(bnwts)
for j in range(len(bnnms)):
nm = bnnms[j]
bns[nm].append(tbns[j])
# average over all batches
for nm in bnnms:
if '_v' in nm:
continue
vnm = nm.replace('_mu', '_v')
mean = np.mean(np.stack(bns[nm], axis=0), axis=0)
mean_var = np.var(np.stack(bns[nm], axis=0), axis=0)
var = np.mean(np.stack(bns[vnm], axis=0), axis=0)
