# if niter > 0:
# mfn = wts+"/iter_%06d.model.npz" % niter
# sfn = wts+"/iter_%06d.state.npz" % niter
# ut.mprint("Restoring model from " + mfn )
# ut.loadNet(mfn,model.weights,sess)
# ut.mprint("Restoring state from " + sfn )
# ut.loadAdam(sfn,opt,model.weights,sess)
# ut.mprint("Done!")
#########################################################################
# Main Training loop
stop = False
ut.mprint("Starting from Iteration %d" % niter)
sess.run(tset.fetchOp, feed_dict=tset.fdict())
while niter < MAXITER and not ut.stop:
## Validate model every so often
if niter % VALFREQ == 0:
ut.mprint("Validating model")
val_iter = vset.ndata // BSZ
vloss, vset.niter = [], 0
sess.run(vset.fetchOp, feed_dict=vset.fdict())
for its in range(val_iter):
sess.run(swpV)
outs = sess.run(lvals + [vset.fetchOp],
feed_dict={
**vset.fdict(), is_training: False
})
if niter > 0:
mfn = wts+"/iter_%06d.model.npz" % niter
sfn = wts+"/iter_%06d.state.npz" % niter
ut.mprint("Restoring model from " + mfn )
ut.loadNet(mfn,model.weights,sess)
ut.mprint("Restoring state from " + sfn )
ut.loadAdam(sfn,opt,model.weights,sess)
ut.mprint("Done!")
#########################################################################
# Main Training loop
stop=False
ut.mprint("Starting from Iteration %d" % niter)
sess.run(tset.fetchOp,feed_dict=tset.fdict())
while niter < MAXITER and not ut.stop:
## Validate model every so often
if niter % VALFREQ == 0:
ut.mprint("Validating model")
val_iter = vset.ndata // BSZ
vloss, vset.niter = [], 0
sess.run(vset.fetchOp,feed_dict=vset.fdict())
for its in range(val_iter):
sess.run(swpV)
outs = sess.run(
lvals+[vset.fetchOp],
feed_dict={**vset.fdict(), is_training: False}
)
