def __init__(self, rng, n_in, n_hidden1, n_hidden2, n_out, activation=sigmoid, activationGradient=sigmoidGradient, learningRate=10**(-5), regularization=0, momentum=0): self.labelRange = np.array(range(10)) self.hiddenLayer1 = HiddenLayer(rng=rng, n_in=n_in, n_out=n_hidden1 + 1, activation=activation, activationGradient=activationGradient) self.hiddenLayer2 = HiddenLayer(rng=rng, n_in=n_hidden1 + 1, n_out=n_hidden2 + 1, activation=activation, activationGradient=activationGradient) self.softmaxTopLayer = SoftmaxTopLayer(rng=rng, n_in=n_hidden2 + 1, n_out=n_out) self.params = [self.hiddenLayer1.params] + [ self.hiddenLayer2.params ] + [self.softmaxTopLayer.params] self.learningRate = learningRate self.regularization = regularization self.momentum = momentum
def __init__(self, rng, n_in, n_hidden1, n_hidden2, n_out, activation = sigmoid, activationGradient = sigmoidGradient, learningRate = 10**(-5), regularization = 0, momentum = 0): self.labelRange = np.array(range(10)) self.hiddenLayer1 = HiddenLayer( rng = rng, n_in = n_in, n_out = n_hidden1 + 1, activation = activation, activationGradient = activationGradient ) self.hiddenLayer2 = HiddenLayer( rng = rng, n_in = n_hidden1 + 1, n_out = n_hidden2 + 1, activation = activation, activationGradient = activationGradient ) self.softmaxTopLayer = SoftmaxTopLayer( rng = rng, n_in = n_hidden2 + 1, n_out = n_out ) self.params = [self.hiddenLayer1.params] + [self.hiddenLayer2.params] + [self.softmaxTopLayer.params] self.learningRate = learningRate self.regularization = regularization self.momentum = momentum
class twoHiddenLayerPerceptron(object): def __init__(self, rng, n_in, n_hidden1, n_hidden2, n_out, activation = sigmoid, activationGradient = sigmoidGradient, learningRate = 10**(-5), regularization = 0, momentum = 0): self.labelRange = np.array(range(10)) self.hiddenLayer1 = HiddenLayer( rng = rng, n_in = n_in, n_out = n_hidden1 + 1, activation = activation, activationGradient = activationGradient ) self.hiddenLayer2 = HiddenLayer( rng = rng, n_in = n_hidden1 + 1, n_out = n_hidden2 + 1, activation = activation, activationGradient = activationGradient ) self.softmaxTopLayer = SoftmaxTopLayer( rng = rng, n_in = n_hidden2 + 1, n_out = n_out ) self.params = [self.hiddenLayer1.params] + [self.hiddenLayer2.params] + [self.softmaxTopLayer.params] self.learningRate = learningRate self.regularization = regularization self.momentum = momentum def forwardPropagate(self, inputValue, output): outputVec = 1*(self.labelRange == output) #propagate to the hidden layer, bias corresponds to a unit with constant ouput 1 self.hiddenLayer1.inputValue = inputValue self.hiddenLayer1.linearOutput = np.dot(self.hiddenLayer1.inputValue, self.hiddenLayer1.W) self.hiddenLayer1.y = self.hiddenLayer1.activation(self.hiddenLayer1.linearOutput) self.hiddenLayer1.y[0] = 1 self.hiddenLayer1.yPrime = self.hiddenLayer1.activationGradient(self.hiddenLayer1.linearOutput) self.hiddenLayer1.yPrime[0] = 0 #propagate to the second hidden layer, bias corresponds to a unit with constant ouput 1 self.hiddenLayer2.inputValue = self.hiddenLayer1.y self.hiddenLayer2.linearOutput = np.dot(self.hiddenLayer2.inputValue, self.hiddenLayer2.W) self.hiddenLayer2.y = self.hiddenLayer2.activation(self.hiddenLayer2.linearOutput) self.hiddenLayer2.y[0] = 1 self.hiddenLayer2.yPrime = self.hiddenLayer2.activationGradient(self.hiddenLayer2.linearOutput) self.hiddenLayer2.yPrime[0] = 0 #propagate to the top layer, bias corresponds to a unit with constant ouput 1 self.softmaxTopLayer.inputValue = self.hiddenLayer2.y self.softmaxTopLayer.linearOutput = np.dot(self.softmaxTopLayer.inputValue, self.softmaxTopLayer.W) self.softmaxTopLayer.y = SoftMaxFunc(self.softmaxTopLayer.linearOutput) self.softmaxTopLayer.predict = np.argmax(self.softmaxTopLayer.y) self.softmaxTopLayer.delta = outputVec - self.softmaxTopLayer.y self.softmaxTopLayer.weightGradient = self.softmaxTopLayer.weightGradient - \ np.outer(self.softmaxTopLayer.inputValue,self.softmaxTopLayer.delta) def backwardPropagate(self): self.hiddenLayer2.delta = np.multiply(self.hiddenLayer2.yPrime, np.dot(self.softmaxTopLayer.W, self.softmaxTopLayer.delta)) self.hiddenLayer2.weightGradient = self.hiddenLayer2.weightGradient - \ np.outer(self.hiddenLayer2.inputValue, self.hiddenLayer2.delta) self.hiddenLayer1.delta = np.multiply(self.hiddenLayer1.yPrime, np.dot(self.hiddenLayer2.W, self.hiddenLayer2.delta)) self.hiddenLayer1.weightGradient = self.hiddenLayer1.weightGradient - \ np.outer(self.hiddenLayer1.inputValue, self.hiddenLayer1.delta) def updateWeight(self): preW = self.softmaxTopLayer.W self.softmaxTopLayer.W = preW - \ self.learningRate*self.softmaxTopLayer.weightGradient + \ 2*self.regularization*preW + self.momentum*self.softmaxTopLayer.deltaW self.softmaxTopLayer.deltaW = self.softmaxTopLayer.W - preW preHiddenW2 = self.hiddenLayer2.W self.hiddenLayer2.W = preHiddenW2 - \ self.learningRate*self.hiddenLayer2.weightGradient + \ 2*self.regularization*preHiddenW2 + self.momentum*self.hiddenLayer2.deltaW self.hiddenLayer2.deltaW = self.hiddenLayer2.W - preHiddenW2 preHiddenW1 = self.hiddenLayer1.W self.hiddenLayer1.W = preHiddenW1 - \ self.learningRate*self.hiddenLayer1.weightGradient + \ 2*self.regularization*preHiddenW1 + self.momentum*self.hiddenLayer1.deltaW self.hiddenLayer1.deltaW = self.hiddenLayer1.W - preHiddenW1 def accuracy(self, INPUT, OUTPUT): count = 0 for inputValue, output in zip(INPUT,OUTPUT): linearOutputHidden1 = np.dot(inputValue, self.hiddenLayer1.W) yHidden1 = self.hiddenLayer1.activation(linearOutputHidden1) yHidden1[0] = 1 linearOutputHidden2 = np.dot(yHidden1, self.hiddenLayer2.W) yHidden2 = self.hiddenLayer2.activation(linearOutputHidden2) yHidden2[0] = 1 linearOutput = np.dot(yHidden2, self.softmaxTopLayer.W) y = SoftMaxFunc(linearOutput) predict = np.argmax(y) if predict == output: count = count + 1 return (float(count) / len(OUTPUT))
class twoHiddenLayerPerceptron(object): def __init__(self, rng, n_in, n_hidden1, n_hidden2, n_out, activation=sigmoid, activationGradient=sigmoidGradient, learningRate=10**(-5), regularization=0, momentum=0): self.labelRange = np.array(range(10)) self.hiddenLayer1 = HiddenLayer(rng=rng, n_in=n_in, n_out=n_hidden1 + 1, activation=activation, activationGradient=activationGradient) self.hiddenLayer2 = HiddenLayer(rng=rng, n_in=n_hidden1 + 1, n_out=n_hidden2 + 1, activation=activation, activationGradient=activationGradient) self.softmaxTopLayer = SoftmaxTopLayer(rng=rng, n_in=n_hidden2 + 1, n_out=n_out) self.params = [self.hiddenLayer1.params] + [ self.hiddenLayer2.params ] + [self.softmaxTopLayer.params] self.learningRate = learningRate self.regularization = regularization self.momentum = momentum def forwardPropagate(self, inputValue, output): outputVec = 1 * (self.labelRange == output) #propagate to the hidden layer, bias corresponds to a unit with constant ouput 1 self.hiddenLayer1.inputValue = inputValue self.hiddenLayer1.linearOutput = np.dot(self.hiddenLayer1.inputValue, self.hiddenLayer1.W) self.hiddenLayer1.y = self.hiddenLayer1.activation( self.hiddenLayer1.linearOutput) self.hiddenLayer1.y[0] = 1 self.hiddenLayer1.yPrime = self.hiddenLayer1.activationGradient( self.hiddenLayer1.linearOutput) self.hiddenLayer1.yPrime[0] = 0 #propagate to the second hidden layer, bias corresponds to a unit with constant ouput 1 self.hiddenLayer2.inputValue = self.hiddenLayer1.y self.hiddenLayer2.linearOutput = np.dot(self.hiddenLayer2.inputValue, self.hiddenLayer2.W) self.hiddenLayer2.y = self.hiddenLayer2.activation( self.hiddenLayer2.linearOutput) self.hiddenLayer2.y[0] = 1 self.hiddenLayer2.yPrime = self.hiddenLayer2.activationGradient( self.hiddenLayer2.linearOutput) self.hiddenLayer2.yPrime[0] = 0 #propagate to the top layer, bias corresponds to a unit with constant ouput 1 self.softmaxTopLayer.inputValue = self.hiddenLayer2.y self.softmaxTopLayer.linearOutput = np.dot( self.softmaxTopLayer.inputValue, self.softmaxTopLayer.W) self.softmaxTopLayer.y = SoftMaxFunc(self.softmaxTopLayer.linearOutput) self.softmaxTopLayer.predict = np.argmax(self.softmaxTopLayer.y) self.softmaxTopLayer.delta = outputVec - self.softmaxTopLayer.y self.softmaxTopLayer.weightGradient = self.softmaxTopLayer.weightGradient - \ np.outer(self.softmaxTopLayer.inputValue,self.softmaxTopLayer.delta) def backwardPropagate(self): self.hiddenLayer2.delta = np.multiply( self.hiddenLayer2.yPrime, np.dot(self.softmaxTopLayer.W, self.softmaxTopLayer.delta)) self.hiddenLayer2.weightGradient = self.hiddenLayer2.weightGradient - \ np.outer(self.hiddenLayer2.inputValue, self.hiddenLayer2.delta) self.hiddenLayer1.delta = np.multiply( self.hiddenLayer1.yPrime, np.dot(self.hiddenLayer2.W, self.hiddenLayer2.delta)) self.hiddenLayer1.weightGradient = self.hiddenLayer1.weightGradient - \ np.outer(self.hiddenLayer1.inputValue, self.hiddenLayer1.delta) def updateWeight(self): preW = self.softmaxTopLayer.W self.softmaxTopLayer.W = preW - \ self.learningRate*self.softmaxTopLayer.weightGradient + \ 2*self.regularization*preW + self.momentum*self.softmaxTopLayer.deltaW self.softmaxTopLayer.deltaW = self.softmaxTopLayer.W - preW preHiddenW2 = self.hiddenLayer2.W self.hiddenLayer2.W = preHiddenW2 - \ self.learningRate*self.hiddenLayer2.weightGradient + \ 2*self.regularization*preHiddenW2 + self.momentum*self.hiddenLayer2.deltaW self.hiddenLayer2.deltaW = self.hiddenLayer2.W - preHiddenW2 preHiddenW1 = self.hiddenLayer1.W self.hiddenLayer1.W = preHiddenW1 - \ self.learningRate*self.hiddenLayer1.weightGradient + \ 2*self.regularization*preHiddenW1 + self.momentum*self.hiddenLayer1.deltaW self.hiddenLayer1.deltaW = self.hiddenLayer1.W - preHiddenW1 def accuracy(self, INPUT, OUTPUT): count = 0 for inputValue, output in zip(INPUT, OUTPUT): linearOutputHidden1 = np.dot(inputValue, self.hiddenLayer1.W) yHidden1 = self.hiddenLayer1.activation(linearOutputHidden1) yHidden1[0] = 1 linearOutputHidden2 = np.dot(yHidden1, self.hiddenLayer2.W) yHidden2 = self.hiddenLayer2.activation(linearOutputHidden2) yHidden2[0] = 1 linearOutput = np.dot(yHidden2, self.softmaxTopLayer.W) y = SoftMaxFunc(linearOutput) predict = np.argmax(y) if predict == output: count = count + 1 return (float(count) / len(OUTPUT))