def RunNeuralNetDriver(nil, maxNetworkIterations, theta, numTrain, numValidation, numTest, improveTheta): #See if a pretrained weight set is given if improveTheta: if theta == 0: print( "!!!ERROR: Theta is set to zero, but improveTheta is set to one. Ser improveTheta to 0 or load a valid theta weight matrix" ) #Train a network, if applicable if theta == 0 or improveTheta: #Load data from MNIST print(">>>Loading Data From MNIST") trainData, validationData, testData, trainLabels, validationLabels, testLabels = GenNeuralNetData.loadMNIST( numTrain, numValidation, numTest) print(">>>Data Loaded") print(">>>Training Neural Network") timer = time.time() theta, validationErrors = TrainNeuralNet.RunTrainNeuralNet( trainData, trainLabels, validationData, validationLabels, nil, maxNetworkIterations, theta, improveTheta) timer = time.time() - timer print(">>>Training Complete. Train Time: " + str(timer) + " Seconds") print(">>>Testing Neural Network Accuracy") timer = time.time() #Calculate accuracy on test set testShape = testData.shape testSamples = testShape[0] testPixels = testShape[1] L = nil.size #Number of layers in FFANN #Initialize misclassifications misclassifications = 0 #Calculate predictions testErr = 0 for m in np.arange(testSamples): #Forward propagation activations = ANNSupport.forwardProp(testData[m, 0:testPixels], testPixels, nil, L, theta) #Backpropagation for final layer to calculate error isIncorrect = ANNSupport.checkPrediction(activations[-1, 0:nil[-1]], testLabels[m], nil, 1) misclassifications += isIncorrect percentAccuracy = (testSamples - misclassifications) / testSamples timer = time.time() - timer print(">>>Testing Complete. Test Time: " + str(timer) + " Seconds. Misclassified " + str(misclassifications) + " Samples. Dataset Size: " + str(testSamples) + " Samples. Test Accuracy: " + str(100 * percentAccuracy) + " Percent.") return theta, validationErrors, percentAccuracy
def RunNeuralNetDriver(nil, maxNetworkIterations, theta, numTrain, numValidation, numTest, improveTheta): #See if a pretrained weight set is given if improveTheta: if theta==0: print("!!!ERROR: Theta is set to zero, but improveTheta is set to one. Ser improveTheta to 0 or load a valid theta weight matrix") #Train a network, if applicable if theta==0 or improveTheta: #Load data from MNIST print(">>>Loading Data From MNIST") trainData, validationData, testData, trainLabels, validationLabels, testLabels = GenNeuralNetData.loadMNIST(numTrain, numValidation, numTest) print(">>>Data Loaded") print(">>>Training Neural Network") timer=time.time() theta, validationErrors = TrainNeuralNet.RunTrainNeuralNet(trainData, trainLabels, validationData, validationLabels, nil, maxNetworkIterations, theta, improveTheta); timer=time.time()-timer print(">>>Training Complete. Train Time: "+str(timer) + " Seconds") print(">>>Testing Neural Network Accuracy") timer=time.time() #Calculate accuracy on test set testShape=testData.shape testSamples=testShape[0] testPixels=testShape[1] L=nil.size #Number of layers in FFANN #Initialize misclassifications misclassifications=0 #Calculate predictions testErr=0 for m in np.arange(testSamples): #Forward propagation activations = ANNSupport.forwardProp(testData[m, 0:testPixels], testPixels, nil, L, theta) #Backpropagation for final layer to calculate error isIncorrect=ANNSupport.checkPrediction(activations[-1, 0:nil[-1]], testLabels[m], nil, 1) misclassifications+=isIncorrect percentAccuracy = (testSamples-misclassifications)/testSamples timer=time.time()-timer print(">>>Testing Complete. Test Time: "+ str(timer) + " Seconds. Misclassified "+str(misclassifications) + " Samples. Dataset Size: " + str(testSamples) + " Samples. Test Accuracy: " + str(100*percentAccuracy) + " Percent.") return theta, validationErrors, percentAccuracy
def RunTrainNeuralNet(trainData, trainLabels, validationData, validationLabels, nil, maxNetworkIterations, theta, improveTheta): trainShape=trainData.shape trainSamples=trainShape[0] trainPixels=trainShape[1] validationShape=validationData.shape validationSamples=validationShape[0] validationPixels=validationShape[1] validationErrors=[] validationWeights=0 validationLookback=2 validationSampleRate=20 L=nil.size #Number of layers. Note that there are L-1 layers in theta alpha=1 #Training constant information alphaTol=0.00000001 if theta==0 and (not improveTheta): theta=np.zeros((np.max(nil[0:-1]), np.max(nil[1:]), L-1)) for n in np.arange(L-1): #Initialize weight layers between -0.5 and 0.5 theta[0:nil[n], 0:nil[n+1], n]=np.random.rand(nil[n], nil[n+1])-0.5 #Initialize activation error matrix (2D). #Note that entries are stored row-wise in activations #and column-wise in deltaLC deltaLC=np.zeros((np.max(nil),L)) #Begin main loop error=999999999 iters=-1 while iters<maxNetworkIterations and error>0.00001 and alpha>alphaTol: iters=iters+1; #Initialize 3D weight update matrix. Must be done each iteration deltaUC=np.zeros(theta.shape) #Reset additional iteration specific variables error=0 print("Iterations: "+str(iters)) #Useful feedback for long training sessions for m in np.arange(trainSamples): #Forward propagation activations = ANNSupport.forwardProp(trainData[m, 0:trainPixels], trainPixels, nil, L, theta) #Backpropagation #Output layer is easy but must be done separately errorVec, squaredError=ANNSupport.checkPrediction(activations[-1, 0:nil[-1]], trainLabels[m], nil, 0) deltaLC[0:nil[L-1],L-1]=errorVec.T deltaUC[0:nil[-2],0:nil[-1],-1] += np.outer(activations[-2, 0:nil[-2]], deltaLC[0:nil[-1],-1]) #Harvest squared error in final layer error += squaredError #Hidden layers. No update to input layer (no error) for layer in np.arange(L-2, 0, -1): activationsInPrevious = activations[layer, 0:nil[layer]] deltaLC[0:nil[layer], layer] = np.dot(theta[0:nil[layer],0:nil[layer+1],layer],deltaLC[0:nil[layer+1],layer+1]).T * activationsInPrevious * (1-activationsInPrevious) deltaUC[0:nil[layer-1],0:nil[layer],layer-1] += np.outer(activations[layer-1, 0:nil[layer-1]], deltaLC[0:nil[layer],layer]) #Scale weights by layer for layer in np.arange(L-1): tempDeltaUCLayer = deltaUC[0:nil[layer],0:nil[layer+1], layer] deltaUC[0:nil[layer],0:nil[layer+1],layer] = tempDeltaUCLayer/np.max(np.abs(tempDeltaUCLayer)) #Update theta. Error always decreases in batch mode done=0 while not done: newTheta = theta - alpha*deltaUC newError=0 for m in np.arange(trainSamples): #Forward propagation activations = ANNSupport.forwardProp(trainData[m, 0:trainPixels], trainPixels, nil, L, newTheta) #Backpropagation for final layer to calculate error errorVec, squaredError=ANNSupport.checkPrediction(activations[-1, 0:nil[-1]], trainLabels[m], nil, 0) newError += squaredError #Implementation of momentum if newError<error: theta=newTheta; alpha*=2; error=newError; done=1; print('new error = '+str(error)) print('alpha = '+str(alpha)); else: #Error has increased alpha/=2 #Evaluation of error on validation set if validationSamples>0 and iters%validationSampleRate==0: validationErr=0 for m in np.arange(validationSamples): #Forward propagation activations = ANNSupport.forwardProp(validationData[m, 0:validationPixels], validationPixels, nil, L, theta) #Backpropagation for final layer to calculate error squaredError=ANNSupport.checkPrediction(activations[-1, 0:nil[-1]], validationLabels[m], nil, 1) validationErr+=squaredError #Validation errors should be saved for early stopping and plotting validationErrors.append(validationErr) #Save the set of weights that performs better on the validations set if len(validationErrors)>1 and validationErrors[-1]<validationErrors[-2]: validationWeights=theta #Check for overfitting if len(validationErrors)>validationLookback and np.max(validationErrors[-validationLookback:])==validationErrors[-1]: #Stop execution print("Overfitting Detected. Returning Weights") break #End of outer iteration loop if iters==maxNetworkIterations: print('Max Iterations Reached. Returning Weights') elif error<=0.00001: print('Error Tolerance Reached') elif alpha<=alphaTol: print('Learning Rate Tolerance Reached') if np.sum(validationWeights)==0: print('Training Weights Returned') return theta, validationErrors else: print('Validation Weights Returned') return validationWeights, validationErrors