Exemplo n.º 1
0
    def computeNumericalGradient(self, _epsilon, X, y):
        """Setting parameters to perform numerical gradient checking"""
        self.Feeder = Feeder(self.mindSingle)
        self.Feeder.randomWeights()
        _paramsInitial = np.hstack((np.array(self.Feeder.inputWeights).ravel(), np.array(self.Feeder.hiddenOneWeights).ravel()))
        self.numgrad = np.zeros(_paramsInitial.shape)
    
        _perturb = np.zeros(_paramsInitial.shape)
     
        for p in range(len(_paramsInitial)):
            _perturb[p] = _epsilon     
            self.resetParams(_paramsInitial + _perturb)
            self.Feeder.feedForward(X, y)
            _loss2 = costFunction(self.Feeder)
                     
            self.resetParams(_paramsInitial - _perturb)
            self.Feeder.feedForward(X, y)
            _loss1 = costFunction(self.Feeder)
                     
            #Computing numericak gradient
            self.numgrad[p] = (_loss2 - _loss1) / (2 * _epsilon) 
            
            #Return the value we changed back to zero
            _perturb[p] = 0

    
        self.resetParams(_paramsInitial)
Exemplo n.º 2
0
 def test_single_randomWeights_hiddenWeights(self):
     self.assign_io_data()
     _neurons = 10
     _mindSingle = Mindy(self.input, self.output, _neurons, 0.1)
     _feed = Feeder(_mindSingle)
     _feed.randomWeights()
     self.assertEqual(_neurons, len(_feed.hiddenOneWeights))
Exemplo n.º 3
0
 def test_single_randomWeights_weightsMatrix(self):
     self.assign_io_data()
     _neurons = 10
     _mindSingle = Mindy(self.input, self.output, _neurons, 0.1)
     _feed = Feeder(_mindSingle)
     _feed.randomWeights()
     self.assertEqual(_neurons * len(self.input.T),
                      len(_feed.inputWeights) * len(_feed.inputWeights.T))
Exemplo n.º 4
0
 def test_single_outputSum_deltaOutputSum(self):
     self.assign_io_data()
     _neurons = 10
     _mindSingle = Mindy(self.input, self.output, _neurons, 0.1)
     _feed = Feeder(_mindSingle)
     _feed.randomWeights()
     _feed.feedForward(_mindSingle.inputM, _mindSingle.outputM)
     _feed.feedBackwards(0.1)
     self.assertEqual(_feed.deltaOutputSum.shape, self.output.shape)
Exemplo n.º 5
0
 def test_single_inputLayerBackward_deltaInputChange(self):
     self.assign_io_data()
     _neurons = 10
     _mindSingle = Mindy(self.input, self.output, _neurons, 0.1)
     _feed = Feeder(_mindSingle)
     _feed.randomWeights()
     _feed.feedForward(_mindSingle.inputM, _mindSingle.outputM)
     _feed.feedBackwards(0.1)
     self.assertEqual(_feed.deltaInputChange.shape,
                      _feed.inputWeights.shape)
Exemplo n.º 6
0
 def test_single_error_oneLower(self):
     self.assign_io_data()
     _neurons = 10
     _mindSingle = Mindy(self.input, self.output, _neurons, 0.1)
     _feed = Feeder(_mindSingle)
     _feed.randomWeights()
     _feed.feedForward(_mindSingle.inputM, _mindSingle.outputM)
     _feed.feedBackwards(0.1)
     _mindSingle.train(100)
     _errors = MindyErrors(_mindSingle)
     self.assertLess(_errors.modelError(), 1)
Exemplo n.º 7
0
 def test_single_train_hiddenWeights(self):
     self.assign_io_data()
     _neurons = 10
     _mindSingle = Mindy(self.input, self.output, _neurons, 0.1)
     _feed = Feeder(_mindSingle)
     _feed.randomWeights()
     _feed.feedForward(_mindSingle.inputM, _mindSingle.outputM)
     _feed.feedBackwards(0.1)
     _mindSingle.train(10)
     self.assertEqual(_feed.hiddenOneWeights.shape,
                      _feed.deltaHiddenChange.shape)
Exemplo n.º 8
0
 def test_single_pushForward_predictedOutput(self):
     self.assign_io_data()
     _neurons = 10
     _mindSingle = Mindy(self.input, self.output, _neurons, 0.1)
     _feed = Feeder(_mindSingle)
     _feed.randomWeights()
     _feed.feedForward(_mindSingle.inputM, _mindSingle.outputM)
     self.assertEqual(_feed.predictedOutput.shape, self.output.shape)
Exemplo n.º 9
0
    def train(self, _iterations):
        """Thinker for the mind""" 
        try:
            if self.ovr:
                self.yCats = self.OVR.yCats
                _i = 0
                while _i < self.yCats:
                    """Adding 3 dimensional matrices as weigths with shape in 3rd dim corresponding to number of categories"""
                    self.Feeder = Feeder(self)
                    self.Feeder.randomWeights()
                    self.outputM = self.OVR.outputOvr[:, _i]
                    self.trainingIterations(_iterations)
                    
                    if _i == 0:
                        self.weightsOvr = np.array(self.Feeder.inputWeights)
                        self.hiddenOneWeightsOvr = np.array(self.Feeder.hiddenOneWeights)
                    
                    elif _i > 0:
                        try: #because stack does not let you add a 2 dim to a 3 dim, but concatenate will. Stack only adds a new dim
                            self.weightsOvr = np.stack((self.weightsOvr, np.array(self.Feeder.inputWeights)), axis=0)
                            self.hiddenOneWeightsOvr = np.stack((self.hiddenOneWeightsOvr, np.array(self.Feeder.hiddenOneWeights)), axis=0)
                        except:
                            self.weightsOvr = np.concatenate((self.weightsOvr, np.array(self.Feeder.inputWeights)[np.newaxis, :]), axis=0)
                            self.hiddenOneWeightsOvr = np.concatenate((self.hiddenOneWeightsOvr, np.array(self.Feeder.hiddenOneWeights)[np.newaxis, :]), axis=0)
                    
                    _i += 1
            if not self.ovr:
                self.Feeder = Feeder(self)
                self.Feeder.randomWeights()
                self.trainingIterations(_iterations)

        except Exception as err:
            traceback.print_exc()

        self.yhat = np.mean(self.Feeder.predictedOutput, axis=0)
        self.residual = np.mean(self.Feeder.residual, axis=0)
Exemplo n.º 10
0
class TestNumericalGradient(unittest.TestCase):
    
    def assign_io_data(self):
        
        
        self.input = np.array([
        [0, 0, 0],
        [1, 1, 1],
        [1, 0, 1],
        [0, 0, 1],
        [1, 1, 1]
        ])
    
        self.output = np.array([
        [0],
        [1],
        [1],
        [0],
        [1]
        ])

    def test_computeNumericalGradient(self):

        self.assign_io_data()
        _neurons = 10
        self.mindSingle = Mindy(self.input, self.output, _neurons, 0.1)
        self.computeNumericalGradient(1e-4, self.mindSingle.inputM, self.mindSingle.outputM)
        _numgrad = self.numgrad
        _grad = self.computeGradients()
        
        
        """VISUAL TEST. CHECK IF THE ARE ALMOST EQUAL"""
        print _numgrad
        print _grad
 
    def computeNumericalGradient(self, _epsilon, X, y):
        """Setting parameters to perform numerical gradient checking"""
        self.Feeder = Feeder(self.mindSingle)
        self.Feeder.randomWeights()
        _paramsInitial = np.hstack((np.array(self.Feeder.inputWeights).ravel(), np.array(self.Feeder.hiddenOneWeights).ravel()))
        self.numgrad = np.zeros(_paramsInitial.shape)
    
        _perturb = np.zeros(_paramsInitial.shape)
     
        for p in range(len(_paramsInitial)):
            _perturb[p] = _epsilon     
            self.resetParams(_paramsInitial + _perturb)
            self.Feeder.feedForward(X, y)
            _loss2 = costFunction(self.Feeder)
                     
            self.resetParams(_paramsInitial - _perturb)
            self.Feeder.feedForward(X, y)
            _loss1 = costFunction(self.Feeder)
                     
            #Computing numericak gradient
            self.numgrad[p] = (_loss2 - _loss1) / (2 * _epsilon) 
            
            #Return the value we changed back to zero
            _perturb[p] = 0

    
        self.resetParams(_paramsInitial)

    def computeGradients(self):
       self.Feeder.feedBackwards(1, _numgrad = True)
       return np.hstack((self.Feeder.deltaInputChange.ravel(), self.Feeder.deltaHiddenChange.ravel()))
        
    
    def resetParams(self, params):
        self.Feeder.inputWeights = np.matrix(params[:-self.mindSingle.neurons]).reshape(len(self.mindSingle.inputM.T), self.mindSingle.neurons)
        self.Feeder.hiddenOneWeights = np.matrix(params[-self.mindSingle.neurons:][:, np.newaxis])
Exemplo n.º 11
0
class Mindy():
    """Class for a simple neural network, Still under construction. NB: Only one hidden layer"""
    def __init__(self, inputData, outputData, neurons, learningRate, **kwargs):
        """initializor, take input and output data and constructs a numpy array, and accepts the amount of intitial neurons"""
        try:
            self.inputM = inputData
            self.outputM = outputData
            self.learningRate = learningRate
        except Exception as err:
            traceback.print_exc()
        try:
            self.neurons = int(neurons)
        except Exception as err:
            traceback.print_exc()
        
        self.yCats = 0 #Index value -> means 1 vector for y
        self.ovr = False
        try:
            for key, value in kwargs.iteritems():
                if key == 'multinomial':
                    if value == 'ovr':
                        self.ovr = True
                        self.OVR = OVR(self)
                        self.OVR.oneVsRestCategorial(np.array(self.outputM))
        except Exception as err:
            traceback.print_exc()
    
    def train(self, _iterations):
        """Thinker for the mind""" 
        try:
            if self.ovr:
                self.yCats = self.OVR.yCats
                _i = 0
                while _i < self.yCats:
                    """Adding 3 dimensional matrices as weigths with shape in 3rd dim corresponding to number of categories"""
                    self.Feeder = Feeder(self)
                    self.Feeder.randomWeights()
                    self.outputM = self.OVR.outputOvr[:, _i]
                    self.trainingIterations(_iterations)
                    
                    if _i == 0:
                        self.weightsOvr = np.array(self.Feeder.inputWeights)
                        self.hiddenOneWeightsOvr = np.array(self.Feeder.hiddenOneWeights)
                    
                    elif _i > 0:
                        try: #because stack does not let you add a 2 dim to a 3 dim, but concatenate will. Stack only adds a new dim
                            self.weightsOvr = np.stack((self.weightsOvr, np.array(self.Feeder.inputWeights)), axis=0)
                            self.hiddenOneWeightsOvr = np.stack((self.hiddenOneWeightsOvr, np.array(self.Feeder.hiddenOneWeights)), axis=0)
                        except:
                            self.weightsOvr = np.concatenate((self.weightsOvr, np.array(self.Feeder.inputWeights)[np.newaxis, :]), axis=0)
                            self.hiddenOneWeightsOvr = np.concatenate((self.hiddenOneWeightsOvr, np.array(self.Feeder.hiddenOneWeights)[np.newaxis, :]), axis=0)
                    
                    _i += 1
            if not self.ovr:
                self.Feeder = Feeder(self)
                self.Feeder.randomWeights()
                self.trainingIterations(_iterations)

        except Exception as err:
            traceback.print_exc()

        self.yhat = np.mean(self.Feeder.predictedOutput, axis=0)
        self.residual = np.mean(self.Feeder.residual, axis=0)

    def trainingIterations(self, _iterations):
        """For training iterations"""
        _i = 0
        while _i < _iterations:
            try:
                self.Feeder.feedForward(self.inputM, self.outputM)
                self.Feeder.feedBackwards(self.learningRate)

            except Exception as err:
                traceback.print_exc()
                break
            _i += 1
             
    def predict(self, _input):
        """Using the weights and predicting the output"""
        if self.ovr:
            _i = 0
            while _i < self.yCats:
                try:
                    _predict = Predictor(self)
                    if _i == 0:
                        _hiddenToOutput = _predict.calcOutput(_input, self.weightsOvr[_i, :, :], self.hiddenOneWeightsOvr[_i, :, :])
                        
                    elif _i > 0:
                        _hiddenToOutput = np.hstack((_hiddenToOutput, _predict.calcOutput(_input, self.weightsOvr[_i, :, :], self.hiddenOneWeightsOvr[_i, :, :])))
                    _i += 1  
                except Exception as err:
                    traceback.print_exc()
                    raise ValueError('Error: %s -Input for prediction does not match number of input variables in the network' %(err)) 
            _predVal = _predict.chooseOvrPrediction(_hiddenToOutput)                  
        #Add the looping to choose the highest value if it is larger than 0.5, otherwise choose 0 sa prediction value
        elif not self.ovr:
            _predict = Predictor(self)
            _hiddenToOutput = _predict.calcOutput(_input, self.Feeder.inputWeights, self.Feeder.hiddenOneWeights)
            _predVal = _predict.chooseBinPrediction(_hiddenToOutput)
        return _predVal