Пример #1
0
    def train(self):
        if not self.resume:
            # cross-validation on original network
            if True:
                valid_error, loss = self._validate()
                assert(_isnum(loss))
            else:
                valid_error, loss = (9999999,9999999)
            self.logger.info("ITER %02d: CROSSVAL PRERUN ERROR %.4f AVG.LOSS %.4f" % (self.iter, 100*valid_error, loss))
            self.loss = loss

        while not self.finished():      
            load(self.model, filename = self.mlp_best, gradients=self.wasAccepted)

            if self.iter >= self.momentum_start:
                moment = self.momentum
            else:
                moment = 0
            if False:
                tr_error, tr_loss = 0, 0
            else:
                tr_error, tr_loss = self._train(learning_rate=self.rate, momentum=moment)
                assert(_isnum(tr_loss))
            self.logger.info("ITER %02d: TRAIN ERROR %03.4f AVG.LOSS %.4f lrate %.6g" % (self.iter, 100*tr_error, tr_loss, self.rate))

            # cross-validation
            valid_error, valid_loss = self._validate()
            self.logger.info("ITER %02d: CROSSVAL ERROR %.4f AVG.LOSS %.4f lrate %.6f" % (self.iter, 100*valid_error, valid_loss, self.rate))

            mlp_next = "%s/it%02drate%4f_Terr%.4f_Tloss%4f_CVerr%4f_CVloss%4f" % (self.wdir, self.iter, self.rate, 100*tr_error, tr_loss, 100*valid_error, valid_loss)

            save(self.model, filename = mlp_next+".tmp")

            if self.halving_criteria == "loss":
                loss_new = valid_loss
            elif self.halving_criteria == "frame_err":
                loss_new = valid_error
            else:
                raise Exception("bad halving_criteria")

            if self.accepted(loss_new, mlp_next):
                os.rename(mlp_next+".tmp", mlp_next)
                self.logger.info("accepted")
            else:
                os.rename(mlp_next+".tmp", mlp_next+"_rejected")
                self.logger.info("rejected")

            self.finalize(loss_new)

        # select the best network
        if self.mlp_best == self.mlp_init:
            self.logger.critical("Error training neural network...")
            #sys.exit(1)

        output_file = os.path.join(self.wdir, self.output_file)
        shutil.copy2(self.mlp_best, output_file)
        self.logger.info("Succeeded training: " + output_file)
Пример #2
0
    def _load_state(self):
        obj = utils.pickle_load(self.wdir + "/layerwisetrainer_state")
        self.layer_index = obj["layer_index"]
        self.iter = obj["iter"]
        self.loss = obj["loss"]
        self.mlp_best = obj["mlp_best"]
        self.mlp_crrnt = obj["mlp_crrnt"]
        #self.iters_without_impr = obj["iters_without_impr"]
        self.train_sets.set_state(obj["train_sets"])

        out = StringIO.StringIO()
        print >> out, "\n********** Resuming from **********"
        print >> out, "layer_index", self.layer_index
        print >> out, "iter", self.iter
        print >> out, "loss", self.loss
        print >> out, "mlp_best", self.mlp_best
        print >> out, "mlp_crrnt", self.mlp_crrnt
        self.logger.info(out.getvalue())

        load(self.model, self.mlp_crrnt, gradients=True)
Пример #3
0
    def _load_state(self):
        obj = utils.pickle_load(self.wdir+"/layerwisetrainer_state")
        self.layer_index = obj["layer_index"]
        self.iter = obj["iter"]
        self.loss = obj["loss"]
        self.mlp_best = obj["mlp_best"]
        self.mlp_crrnt = obj["mlp_crrnt"]
        #self.iters_without_impr = obj["iters_without_impr"]
        self.train_sets.set_state(obj["train_sets"])

        out = StringIO.StringIO()
        print >>out, "\n********** Resuming from **********"
        print >>out, "layer_index", self.layer_index
        print >>out, "iter", self.iter
        print >>out, "loss", self.loss
        print >>out, "mlp_best", self.mlp_best
        print >>out, "mlp_crrnt", self.mlp_crrnt
        self.logger.info(out.getvalue())

        load(self.model, self.mlp_crrnt, gradients=True)
Пример #4
0
    def train(self):
        if not self.resume:
            # cross-validation on original network
            if True:
                valid_error, loss = self._validate()
                assert (_isnum(loss))
            else:
                valid_error, loss = (9999999, 9999999)
            self.logger.info(
                "ITER %02d: CROSSVAL PRERUN ERROR %.4f AVG.LOSS %.4f" %
                (self.iter, 100 * valid_error, loss))
            self.loss = loss

        while not self.finished():
            load(self.model,
                 filename=self.mlp_best,
                 gradients=self.wasAccepted)

            if self.iter >= self.momentum_start:
                moment = self.momentum
            else:
                moment = 0
            if False:
                tr_error, tr_loss = 0, 0
            else:
                tr_error, tr_loss = self._train(learning_rate=self.rate,
                                                momentum=moment)
                assert (_isnum(tr_loss))
            self.logger.info(
                "ITER %02d: TRAIN ERROR %03.4f AVG.LOSS %.4f lrate %.6g" %
                (self.iter, 100 * tr_error, tr_loss, self.rate))

            # cross-validation
            valid_error, valid_loss = self._validate()
            self.logger.info(
                "ITER %02d: CROSSVAL ERROR %.4f AVG.LOSS %.4f lrate %.6f" %
                (self.iter, 100 * valid_error, valid_loss, self.rate))

            mlp_next = "%s/it%02drate%4f_Terr%.4f_Tloss%4f_CVerr%4f_CVloss%4f" % (
                self.wdir, self.iter, self.rate, 100 * tr_error, tr_loss,
                100 * valid_error, valid_loss)

            save(self.model, filename=mlp_next + ".tmp")

            if self.halving_criteria == "loss":
                loss_new = valid_loss
            elif self.halving_criteria == "frame_err":
                loss_new = valid_error
            else:
                raise Exception("bad halving_criteria")

            if self.accepted(loss_new, mlp_next):
                os.rename(mlp_next + ".tmp", mlp_next)
                self.logger.info("accepted")
            else:
                os.rename(mlp_next + ".tmp", mlp_next + "_rejected")
                self.logger.info("rejected")

            self.finalize(loss_new)

        # select the best network
        if self.mlp_best == self.mlp_init:
            self.logger.critical("Error training neural network...")
            #sys.exit(1)

        output_file = os.path.join(self.wdir, self.output_file)
        shutil.copy2(self.mlp_best, output_file)
        self.logger.info("Succeeded training: " + output_file)