ut.mprint("Done!") else: # Load the last model in pretraining wcard = os.path.dirname(wts)+"/pretrain/iter_*.model.npz" lst=[(l,int(re.match('.*/.*_(\d+)',l).group(1))) for l in glob(wcard)] mfn = max(lst, key=lambda x: x[1])[0] ut.mprint("Start with pre-trained model " + mfn ) ut.loadNet(mfn,model,sess) ######################################################################### # Main Training loop stop=False ut.mprint("Starting from Iteration %d" % niter) sess.run(tset.fetchOp,feed_dict=tset.fdict()) ## Learning rate, if resume at an iteration after drop if niter >= 4e5 and niter < 5e5: sess.run(tf.assign(lr, LR/np.sqrt(10.0))) elif niter == 5e5: sess.run(tf.assign(lr, LR/10.0)) while niter < MAXITER and not ut.stop: ## Validate model every so often if niter % VALFREQ == 0: ut.mprint("Validating model") val_iter = vset.ndata // BSZ vloss, vset.niter = 0., 0 sess.run(vset.fetchOp,feed_dict=vset.fdict())
intra_op_parallelism_threads=int(nthr))) sess.run(tf.global_variables_initializer()) # Load model wts = np.load(mfile) ph = tf.placeholder(tf.float32) for k in model.weights.keys(): wvar = model.weights[k] wk = wts[k].reshape(wvar.get_shape()) sess.run(wvar.assign(ph), feed_dict={ph: wk}) # Get population stat bnnms, bnwts = list(model.bnwts.keys()), list(model.bnwts.values()) bns = {k: [] for k in bnnms} for i in range(niter): sess.run(tset.fetchOp, feed_dict=tset.fdict()) tbns = sess.run(bnwts) for j in range(len(bnnms)): nm = bnnms[j] bns[nm].append(tbns[j]) # average over all batches for nm in bnnms: if '_v' in nm: continue vnm = nm.replace('_mu', '_v') mean = np.mean(np.stack(bns[nm], axis=0), axis=0) mean_var = np.var(np.stack(bns[nm], axis=0), axis=0) var = np.mean(np.stack(bns[vnm], axis=0), axis=0)