def train(self):
if not self.resume:
# cross-validation on original network
if True:
valid_error, loss = self._validate()
assert(_isnum(loss))
else:
valid_error, loss = (9999999,9999999)
self.logger.info("ITER %02d: CROSSVAL PRERUN ERROR %.4f AVG.LOSS %.4f" % (self.iter, 100*valid_error, loss))
self.loss = loss
while not self.finished():
load(self.model, filename = self.mlp_best, gradients=self.wasAccepted)
if self.iter >= self.momentum_start:
moment = self.momentum
else:
moment = 0
if False:
tr_error, tr_loss = 0, 0
else:
tr_error, tr_loss = self._train(learning_rate=self.rate, momentum=moment)
assert(_isnum(tr_loss))
self.logger.info("ITER %02d: TRAIN ERROR %03.4f AVG.LOSS %.4f lrate %.6g" % (self.iter, 100*tr_error, tr_loss, self.rate))
# cross-validation
valid_error, valid_loss = self._validate()
self.logger.info("ITER %02d: CROSSVAL ERROR %.4f AVG.LOSS %.4f lrate %.6f" % (self.iter, 100*valid_error, valid_loss, self.rate))
mlp_next = "%s/it%02drate%4f_Terr%.4f_Tloss%4f_CVerr%4f_CVloss%4f" % (self.wdir, self.iter, self.rate, 100*tr_error, tr_loss, 100*valid_error, valid_loss)
save(self.model, filename = mlp_next+".tmp")
if self.halving_criteria == "loss":
loss_new = valid_loss
elif self.halving_criteria == "frame_err":
loss_new = valid_error
else:
raise Exception("bad halving_criteria")
if self.accepted(loss_new, mlp_next):
os.rename(mlp_next+".tmp", mlp_next)
self.logger.info("accepted")
else:
os.rename(mlp_next+".tmp", mlp_next+"_rejected")
self.logger.info("rejected")
self.finalize(loss_new)
# select the best network
if self.mlp_best == self.mlp_init:
self.logger.critical("Error training neural network...")
#sys.exit(1)
output_file = os.path.join(self.wdir, self.output_file)
shutil.copy2(self.mlp_best, output_file)
self.logger.info("Succeeded training: " + output_file)
def _load_state(self):
obj = utils.pickle_load(self.wdir + "/layerwisetrainer_state")
self.layer_index = obj["layer_index"]
self.iter = obj["iter"]
self.loss = obj["loss"]
self.mlp_best = obj["mlp_best"]
self.mlp_crrnt = obj["mlp_crrnt"]
#self.iters_without_impr = obj["iters_without_impr"]
self.train_sets.set_state(obj["train_sets"])
out = StringIO.StringIO()
print >> out, "\n********** Resuming from **********"
print >> out, "layer_index", self.layer_index
print >> out, "iter", self.iter
print >> out, "loss", self.loss
print >> out, "mlp_best", self.mlp_best
print >> out, "mlp_crrnt", self.mlp_crrnt
self.logger.info(out.getvalue())
load(self.model, self.mlp_crrnt, gradients=True)
def _load_state(self):
obj = utils.pickle_load(self.wdir+"/layerwisetrainer_state")
self.layer_index = obj["layer_index"]
self.iter = obj["iter"]
self.loss = obj["loss"]
self.mlp_best = obj["mlp_best"]
self.mlp_crrnt = obj["mlp_crrnt"]
#self.iters_without_impr = obj["iters_without_impr"]
self.train_sets.set_state(obj["train_sets"])
out = StringIO.StringIO()
print >>out, "\n********** Resuming from **********"
print >>out, "layer_index", self.layer_index
print >>out, "iter", self.iter
print >>out, "loss", self.loss
print >>out, "mlp_best", self.mlp_best
print >>out, "mlp_crrnt", self.mlp_crrnt
self.logger.info(out.getvalue())
load(self.model, self.mlp_crrnt, gradients=True)
def train(self):
if not self.resume:
# cross-validation on original network
if True:
valid_error, loss = self._validate()
assert (_isnum(loss))
else:
valid_error, loss = (9999999, 9999999)
self.logger.info(
"ITER %02d: CROSSVAL PRERUN ERROR %.4f AVG.LOSS %.4f" %
(self.iter, 100 * valid_error, loss))
self.loss = loss
while not self.finished():
load(self.model,
filename=self.mlp_best,
gradients=self.wasAccepted)
if self.iter >= self.momentum_start:
moment = self.momentum
else:
moment = 0
if False:
tr_error, tr_loss = 0, 0
else:
tr_error, tr_loss = self._train(learning_rate=self.rate,
momentum=moment)
assert (_isnum(tr_loss))
self.logger.info(
"ITER %02d: TRAIN ERROR %03.4f AVG.LOSS %.4f lrate %.6g" %
(self.iter, 100 * tr_error, tr_loss, self.rate))
# cross-validation
valid_error, valid_loss = self._validate()
self.logger.info(
"ITER %02d: CROSSVAL ERROR %.4f AVG.LOSS %.4f lrate %.6f" %
(self.iter, 100 * valid_error, valid_loss, self.rate))
mlp_next = "%s/it%02drate%4f_Terr%.4f_Tloss%4f_CVerr%4f_CVloss%4f" % (
self.wdir, self.iter, self.rate, 100 * tr_error, tr_loss,
100 * valid_error, valid_loss)
save(self.model, filename=mlp_next + ".tmp")
if self.halving_criteria == "loss":
loss_new = valid_loss
elif self.halving_criteria == "frame_err":
loss_new = valid_error
else:
raise Exception("bad halving_criteria")
if self.accepted(loss_new, mlp_next):
os.rename(mlp_next + ".tmp", mlp_next)
self.logger.info("accepted")
else:
os.rename(mlp_next + ".tmp", mlp_next + "_rejected")
self.logger.info("rejected")
self.finalize(loss_new)
# select the best network
if self.mlp_best == self.mlp_init:
self.logger.critical("Error training neural network...")
#sys.exit(1)
output_file = os.path.join(self.wdir, self.output_file)
shutil.copy2(self.mlp_best, output_file)
self.logger.info("Succeeded training: " + output_file)