Exemplo n.º 1
0
def evalRNNOnSeqClassificationDataset(net, testing_dataset, verbose = False, silent = False):
   
    # Fetch targets and calculate the modules output on dataset.
    # Output and target are in one-of-many format. The class for each sequence is
    # determined by first summing the probabilities for each individual sample over
    # the sequence, and then finding its maximum.
    target = testing_dataset.getField("target")
    
    outputs = []
    
    # print net
    
    for seq in testing_dataset._provideSequences():
        net.reset()
        # print 'seq:'
        # print seq
        for i in xrange(len(seq)):
            output = net.activate(seq[i][0])
            outputs.append(output.copy())
    outputs = array(outputs)
    
    

    # determine last indices of the sequences inside dataset
    ends = SequenceHelper.getSequenceEnds(testing_dataset)
    ##format = "%d"*len(ends)
    summed_output = zeros(testing_dataset.outdim)
    # class_output and class_target will store class labels instead of
    # one-of-many values
    class_output = []
    class_target = []
    for j in xrange(len(outputs)):
        # sum up the output values of one sequence
        # print outputs[j]
        summed_output += outputs[j]
#            print j, output[j], " --> ", summed_output
        # if we reached the end of the sequence
        if j in ends:
            # print '------------------------------------------'
            # convert summed_output and target to class labels
            class_output.append(argmax(summed_output))
            class_target.append(argmax(target[j]))

            # reset the summed_output to zeros
            summed_output = zeros(testing_dataset.outdim)

    ##print format % tuple(class_output)
    ##print format % tuple(class_target)

    class_output = array(class_output)
    class_target = array(class_target)
#    print class_target
#    print class_output
    accuracy =  Validator.classificationPerformance(class_output, class_target)
    return (class_output, class_target, accuracy)
Exemplo n.º 2
0
    def dataset_eval(dataset):
        """Return dataset hit rate and MSE"""
        # Transform output values to bit vectors, similar to the targets
        predicted = bit_array_transform(ff_network.activate(x)
                                        for x in dataset['input'])
        target = dataset['target']

        # Lists of positions holding predicted and target classes to compare
        predicted_pos = [list(x).index(1) for x in predicted]
        target_pos = [list(x).index(1) for x in target]

        hits = Validator.classificationPerformance(predicted_pos, target_pos)
        mse = Validator.MSE(predicted, target)
        return hits, mse
Exemplo n.º 3
0
def testOnSequenceData(module, dataset):
    """
    Fetch targets and calculate the modules output on dataset.
    Output and target are in one-of-many format. The class for each sequence is
    determined by argmax OF THE LAST ITEM IN THE SEQUENCE.
    """
    target = dataset.getField("target")
    output = ModuleValidator.calculateModuleOutput(module, dataset)

    # determine last indices of the sequences inside dataset
    ends = SequenceHelper.getSequenceEnds(dataset)

    class_output = array([argmax(output[end]) for end in ends])
    class_target = array([argmax(target[end]) for end in ends])

    return Validator.classificationPerformance(class_output, class_target)
Exemplo n.º 4
0
    def runTraining(self, convergence=0, **kwargs):
        """ Trains the network on the stored dataset. If convergence is >0, check after that many epoch increments
        whether test error is going down again, and stop training accordingly. """
        assert isinstance(self.Trainer, Trainer)
        if self.Graph is not None:
            self.Graph.setLabels(x='epoch', y='% classification error')
            self.Graph.setLegend(['training','test'],loc='lower right')
        epoch = 0
        inc = self.epoinc
        best_error = 100.0
        best_epoch = 0
        learncurve_x = [0]
        learncurve_y = [0.0]
        valcurve_y = [0.0]
        converged = False
        convtest = 0
        if convergence>0:
            logging.info("Convergence criterion: %d batches of %d epochs w/o improvement" % (convergence, inc))
        while epoch<=self.maxepochs and not converged:
            self.Trainer.trainEpochs(inc)
            epoch+=inc
            learncurve_x.append(epoch)
            # calculate errors on TRAINING data
            if isinstance(self.DS, SequentialDataSet):
                r_trn = 100. * (1.0-testOnSequenceData(self.Trainer.module, self.DS))
            else:
                # FIXME: messy - validation does not belong into the Trainer...
                out, trueclass = self.Trainer.testOnClassData(return_targets=True)
                r_trn = 100. * (1.0-Validator.classificationPerformance(out, trueclass))
            learncurve_y.append(r_trn)
            if self.TDS is None:
                logging.info("epoch: %6d,  err_trn: %5.2f%%" % (epoch, r_trn))
            else:
                # calculate errors on TEST data
                if isinstance(self.DS, SequentialDataSet):
                    r_tst = 100. * (1.0-testOnSequenceData(self.Trainer.module, self.TDS))
                else:
                    # FIXME: messy - validation does not belong into the Trainer...
                    out, trueclass = self.Trainer.testOnClassData(return_targets=True, dataset=self.TDS)
                    r_tst = 100. * (1.0-Validator.classificationPerformance(out, trueclass))
                valcurve_y.append(r_tst)
                if r_tst < best_error:
                    best_epoch = epoch
                    best_error = r_tst
                    bestweights = self.Trainer.module.params.copy()
                    convtest = 0
                else:
                    convtest += 1
                logging.info("epoch: %6d,  err_trn: %5.2f%%,  err_tst: %5.2f%%,  best_tst: %5.2f%%" % (epoch, r_trn, r_tst, best_error))
                if self.Graph is not None:
                    self.Graph.addData(1, epoch, r_tst)
                    
                # check if convegence criterion is fulfilled (no improvement after N epoincs)
                if convtest >= convergence:
                    converged = True
                    
            if self.Graph is not None:
                self.Graph.addData(0, epoch, r_trn)
                self.Graph.update()

        logging.info("Best epoch: %6d, with error: %5.2f%%" % (best_epoch, best_error))
        if self.VDS is not None:
            # calculate errors on VALIDATION data
            self.Trainer.module.params[:] = bestweights.copy()
            if isinstance(self.DS, SequentialDataSet):
                r_val = 100. * (1.0-testOnSequenceData(self.Trainer.module, self.VDS))
            else:
                out, trueclass = self.Trainer.testOnClassData(return_targets=True, dataset=self.VDS)
                r_val = 100. * (1.0-Validator.classificationPerformance(out, trueclass))
            logging.info("Result on evaluation data: %5.2f%%" % r_val)
            
        self.trainCurve = (learncurve_x, learncurve_y, valcurve_y)
Exemplo n.º 5
0
    def runTraining(self, convergence=0, **kwargs):
        """ Trains the network on the stored dataset. If convergence is >0, check after that many epoch increments
        whether test error is going down again, and stop training accordingly. """
        assert isinstance(self.Trainer, Trainer)
        if self.Graph is not None:
            self.Graph.setLabels(x='epoch', y='% classification error')
            self.Graph.setLegend(['training','test'],loc='lower right')
        epoch = 0
        inc = self.epoinc
        best_error = 100.0
        best_epoch = 0
        learncurve_x = [0]
        learncurve_y = [0.0]
        valcurve_y = [0.0]
        converged = False
        convtest = 0
        if convergence>0:
            logging.info("Convergence criterion: %d batches of %d epochs w/o improvement" % (convergence, inc))
        while epoch<=self.maxepochs and not converged:
            self.Trainer.trainEpochs(inc)
            epoch+=inc
            learncurve_x.append(epoch)
            # calculate errors on TRAINING data
            if isinstance(self.DS, SequentialDataSet):
                r_trn = 100. * (1.0-testOnSequenceData(self.Trainer.module, self.DS))
            else:
                # FIXME: messy - validation does not belong into the Trainer...
                out, trueclass = self.Trainer.testOnClassData(return_targets=True)
                r_trn = 100. * (1.0-Validator.classificationPerformance(out, trueclass))
            learncurve_y.append(r_trn)
            if self.TDS is None:
                logging.info("epoch: %6d,  err_trn: %5.2f%%" % (epoch, r_trn))
            else:
                # calculate errors on TEST data
                if isinstance(self.DS, SequentialDataSet):
                    r_tst = 100. * (1.0-testOnSequenceData(self.Trainer.module, self.TDS))
                else:
                    # FIXME: messy - validation does not belong into the Trainer...
                    out, trueclass = self.Trainer.testOnClassData(return_targets=True, dataset=self.TDS)
                    r_tst = 100. * (1.0-Validator.classificationPerformance(out, trueclass))
                valcurve_y.append(r_tst)
                if r_tst < best_error:
                    best_epoch = epoch
                    best_error = r_tst
                    bestweights = self.Trainer.module.params.copy()
                    convtest = 0
                else:
                    convtest += 1
                logging.info("epoch: %6d,  err_trn: %5.2f%%,  err_tst: %5.2f%%,  best_tst: %5.2f%%" % (epoch, r_trn, r_tst, best_error))
                if self.Graph is not None:
                    self.Graph.addData(1, epoch, r_tst)
                    
                # check if convegence criterion is fulfilled (no improvement after N epoincs)
                if convtest >= convergence:
                    converged = True
                    
            if self.Graph is not None:
                self.Graph.addData(0, epoch, r_trn)
                self.Graph.update()

        logging.info("Best epoch: %6d, with error: %5.2f%%" % (best_epoch, best_error))
        if self.VDS is not None:
            # calculate errors on VALIDATION data
            self.Trainer.module.params[:] = bestweights.copy()
            if isinstance(self.DS, SequentialDataSet):
                r_val = 100. * (1.0-testOnSequenceData(self.Trainer.module, self.VDS))
            else:
                out, trueclass = self.Trainer.testOnClassData(return_targets=True, dataset=self.VDS)
                r_val = 100. * (1.0-Validator.classificationPerformance(out, trueclass))
            logging.info("Result on evaluation data: %5.2f%%" % r_val)
            
        self.trainCurve = (learncurve_x, learncurve_y, valcurve_y)