print n_in, "n_in" upLayer2 = HiddenLayer(rng, input=upLayer2_input, n_in=nkerns[1] * imageSize2 * imageSize2, n_out=n_out_layer2) upLayer2.W.set_value(W2,borrow=True) upLayer2.b.set_value(b2,borrow=True) upLayer3 = LogisticRegression(input=upLayer2.out(), n_in=n_out_layer2, n_out=7) upLayer3.W.set_value(W3,borrow=True) upLayer3.b.set_value(b3,borrow=True) downLayer3 = LogisticRegression(input=upLayer3.out(), n_in=7, n_out=n_out_layer2) # ##TODO: transpose or inverse W3Down = numpy.transpose(W3) W3Down = W3Down[::-1,::-1] # # downLayer3.W.set_value(W3Down,borrow=True) downLayer3.b.set_value(b3,borrow=True) # # ## construct a fully-connected sigmoidal layer downLayer2 = HiddenLayer(rng, input=upLayer2.out(), n_in=nkerns[1] * imageSize2 * imageSize2, n_out=n_out_layer2)
def eval_param(run=0): #adjust parameters if trained with dropout filename = '../learnedData/weights/cnn_tanh_15_10_15_15_50_015_100epochs_'+str(run)+'.txt' with open(filename, "r") as fp: W3 = cPickle.load(fp) b3 = cPickle.load(fp) W2 = cPickle.load(fp) b2 = cPickle.load(fp) W1 = cPickle.load(fp) b1 = cPickle.load(fp) W0 = cPickle.load(fp) b0 = cPickle.load(fp) if dropout: dropParams = numpy.asarray(cPickle.load(fp)) W3 = numpy.asarray(W3) b3 = numpy.asarray(b3) W2 = numpy.asarray(W2) b2 = numpy.asarray(b2) W1 = numpy.asarray(W1) b1 = numpy.asarray(b1) W0 = numpy.asarray(W0) b0 = numpy.asarray(b0) print W0.shape print b0.shape print W1.shape print b1.shape print W2.shape print b2.shape print W3.shape print b3.shape if dropout: W3 = W3 * (1.0-dropParams[3]) W2 = W2 * (1.0-dropParams[2]) W1 = W1 * (1.0-dropParams[1]) W0 = W0 * (1.0-dropParams[0]) # print numpy.min(W0), numpy.max(W0), numpy.min(W1), numpy.max(W1), numpy.min(W2), numpy.max(W2) batch_size = 1 # get guess for one image imageSize = 50; poolSize = 2; numberClasses = 7; fullyConnectedNetwork = W2.shape[1]; # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch x = T.matrix('x') # the data is presented as rasterized images y = T.ivector('y') # the labels are presented as 1D vector of rng = numpy.random.RandomState(23455) nkerns = [W0.shape[0],W1.shape[0]] layer0_input = x.reshape((batch_size, 1, imageSize, imageSize)) layer0 = LeNetConvPoolLayer(rng, input=layer0_input, image_shape=(batch_size, 1, imageSize, imageSize), filter_shape=W0.shape, poolsize=(poolSize, poolSize), test=False) layer0.W.set_value(W0,borrow=True) layer0.b.set_value(b0,borrow=True) # Construct the second convolutional pooling layer imageSize1 = (imageSize - W0.shape[2] + 1)/poolSize #correct image shape index layer1 = LeNetConvPoolLayer(rng, input=layer0.output, image_shape=(batch_size, nkerns[0], imageSize1, imageSize1), filter_shape=W1.shape, poolsize=(poolSize, poolSize), test=False) layer1.W.set_value(W1,borrow=True) layer1.b.set_value(b1,borrow=True) # the HiddenLayer being fully-connected, it operates on 2D matrices of imageSize2 = (imageSize1 - W1.shape[2] + 1)/poolSize layer2_input = layer1.output.flatten(2) # construct a fully-connected sigmoidal layer layer2 = HiddenLayer(rng, input=layer2_input, n_in=nkerns[1] * imageSize2 * imageSize2, n_out=fullyConnectedNetwork, activation=T.tanh) layer2.W.set_value(W2,borrow=True) layer2.b.set_value(b2,borrow=True) # classify the values of the fully-connected sigmoidal layer layer3 = LogisticRegression(input=layer2.output, n_in=fullyConnectedNetwork, n_out=numberClasses) layer3.W.set_value(W3,borrow=True) layer3.b.set_value(b3,borrow=True) # the cost we minimize during training is the NLL of the model cost = layer3.negative_log_likelihood(y) if augmented: datasets = load_data() else: datasets = load_standard_data() train_set_x, train_set_y = datasets[0] valid_set_x, valid_set_y = datasets[1] test_set_x, test_set_y = datasets[2] #print test_set_y.eval().shape n_test_batches = test_set_y.eval().shape[0] n_valid_batches = valid_set_y.eval().shape[0] n_train_batches = train_set_y.eval().shape[0] # create a function to compute the mistakes on the training trained_model = theano.function([index], layer3.errors(y), givens={ x: train_set_x[index * batch_size: (index + 1) * batch_size], y: train_set_y[index * batch_size: (index + 1) * batch_size]}) # create a function to compute the mistakes that are made by the model test_model = theano.function([index], layer3.errors(y), givens={ x: test_set_x[index * batch_size: (index + 1) * batch_size], y: test_set_y[index * batch_size: (index + 1) * batch_size]}) # create a function to compute the mistakes that are made by the model valid_model = theano.function([index], layer3.errors(y), givens={ x: valid_set_x[index * batch_size: (index + 1) * batch_size], y: valid_set_y[index * batch_size: (index + 1) * batch_size]}) # function for guessing the expressions of the test set output_test = theano.function([index], layer3.out(), givens={ x: test_set_x[index * batch_size: (index + 1) * batch_size]}) train_losses = [trained_model(i) for i in xrange(n_train_batches)] train_score = numpy.mean(train_losses) print((' train error rate of model for %d train set: %f %%') % (n_train_batches, train_score * 100.)) # test it on the test set test_losses = [test_model(i) for i in xrange(n_test_batches)] test_score = numpy.mean(test_losses) print((' test error rate of model for %d testcases: %f %%') % (n_test_batches, test_score * 100.)) valid_losses = [valid_model(i) for i in xrange(n_valid_batches)] valid_score = numpy.mean(valid_losses) print((' valid error rate of model for %d valid set: %f %%') % (n_valid_batches, valid_score * 100.)) #for index in range(n_test_batches): # print 'index ', index, 'guess ', output_test(index),'true value ', test_set_y[index * batch_size: (index + 1) * batch_size].eval() confusionMatrix = numpy.zeros((numberClasses,numberClasses)) #print test_set_y[1].eval() for i in range(n_test_batches): #print test_set_y[i].eval(), output_test(i)[0] confusionMatrix[int(test_set_y[i].eval())][int(output_test(i)[0])] = confusionMatrix[int(test_set_y[i].eval())][int(output_test(i)[0])] + 1 #print confusionMatrix[test_set_y[i].eval()][output_test(i)[0]] print confusionMatrix #for index in range(20): # print output_test(index), ' output guess for picture number: ', index # visualize(test_set_x[index * batch_size: (index + 1) * batch_size].eval()) # # confusionMatrix + number of examples per class confusionMatrixPlus = numpy.zeros((numberClasses,numberClasses+1)) casesPerClass = numpy.sum(confusionMatrix, axis=1) # confusion matrix with true positive rate # confusionMatrixPlus[:,:-2] = confusionMatrix # confusionMatrixPlus[:,-2] = casesPerClass # truePositives = confusionMatrix.diagonal()/casesPerClass # confusionMatrixPlus[:,-1] = truePositives # print confusionMatrixPlus # OA = numpy.trace(confusionMatrix)/n_test_batches # #print OA # OA_error = 1.0 - OA # #print OA_error, 'unbalanced error' # BA = 0.0 # for i in range(numberClasses): # BA = confusionMatrixPlus[i,i]/confusionMatrixPlus[i,-2] + BA confusionMatrixPlus[:,:-1] = confusionMatrix confusionMatrixPlus[:,-1] = casesPerClass print confusionMatrixPlus OA = numpy.trace(confusionMatrix)/n_test_batches #print OA OA_error = 1.0 - OA #print OA_error, 'unbalanced error' BA = 0.0 for i in range(numberClasses): BA = confusionMatrixPlus[i,i]/confusionMatrixPlus[i,-1] + BA BA = BA/numberClasses BA_error = 1.0 - BA #print BA_error, 'balanced error' print((' unbalanced error rate of model for %d test cases: %f %%') % (n_test_batches, OA_error * 100.)) print((' balanced error rate of model for %d test cases: %f %%') % (n_test_batches, BA_error * 100.)) print filename return BA_error, OA_error, train_score, valid_score
def eval_param(run=0): filename = '../learnedData/weights/bigcnn_relu_cropping_flipping_45_25_15_5_5_5_100_100_06_100epochs_'+str(run)+'.txt' with open(filename, "r") as fp: W5 = cPickle.load(fp) b5 = cPickle.load(fp) W4 = cPickle.load(fp) b4 = cPickle.load(fp) W3 = cPickle.load(fp) b3 = cPickle.load(fp) W2 = cPickle.load(fp) b2 = cPickle.load(fp) W1 = cPickle.load(fp) b1 = cPickle.load(fp) W0 = cPickle.load(fp) b0 = cPickle.load(fp) if dropout: dropParams = numpy.asarray(cPickle.load(fp)) #fp.close() batch_size = 1 # get guess for one image # # cast in case it is a CudaNdarray W5 = numpy.asarray(W5) b5 = numpy.asarray(b5) W4 = numpy.asarray(W4) b4 = numpy.asarray(b4) W3 = numpy.asarray(W3) b3 = numpy.asarray(b3) W2 = numpy.asarray(W2) b2 = numpy.asarray(b2) W1 = numpy.asarray(W1) b1 = numpy.asarray(b1) W0 = numpy.asarray(W0) b0 = numpy.asarray(b0) print W0.shape print b0.shape print W1.shape print b1.shape print W2.shape print b2.shape print W3.shape print b3.shape print W4.shape print b4.shape print W5.shape print b5.shape if dropout: W4 = W4 * (1.0-dropParams[4]) W3 = W3 * (1.0-dropParams[3]) W2 = W2 * (1.0-dropParams[2]) W1 = W1 * (1.0-dropParams[1]) W0 = W0 * (1.0-dropParams[0]) imageSize = 50; poolSize = 2; numberClasses = 7; fullyConnectedNetwork1 = W2.shape[1]; fullyConnectedNetwork2 = W3.shape[1]; # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch x = T.matrix('x') # the data is presented as rasterized images y = T.ivector('y') # the labels are presented as 1D vector of rng = numpy.random.RandomState(23455) nkerns = [W0.shape[0],W1.shape[0],W2.shape[0]] layer0_input = x.reshape((batch_size, 1, imageSize, imageSize)) # Construct the first convolutional pooling layer layer0 = LeNetConvPoolLayerRelu(rng, input=layer0_input, image_shape=(batch_size, 1, imageSize, imageSize), filter_shape=W0.shape, poolsize=(poolSize, poolSize), test=True) layer0.W.set_value(W0,borrow=True) layer0.b.set_value(b0,borrow=True) imageSize1 = (imageSize - W0.shape[2] + 1)/poolSize # Construct the second convolutional pooling layer layer1 = LeNetConvPoolLayerRelu(rng, input=layer0.output, image_shape=(batch_size, nkerns[0], imageSize1, imageSize1), filter_shape=W1.shape, poolsize=(poolSize, poolSize), test=True) layer1.W.set_value(W1,borrow=True) layer1.b.set_value(b1,borrow=True) imageSize2 = (imageSize1 - W1.shape[2] + 1)/poolSize #correct image shape index # Construct the third convolutional pooling layer layer2 = LeNetConvPoolLayerRelu(rng, input=layer1.output, image_shape=(batch_size, nkerns[1], imageSize2, imageSize2), filter_shape=W2.shape, poolsize=(poolSize, poolSize), test=True) layer2.W.set_value(W2,borrow=True) layer2.b.set_value(b2,borrow=True) # the HiddenLayer being fully-connected, it operates on 2D matrices of # shape (batch_size,num_pixels) (i.e matrix of rasterized images). # This will generate a matrix of shape (1,nkerns[2] * imageSize3 * imageSize3) layer3_input = layer2.output.flatten(2) imageSize3 = (imageSize2 - W2.shape[2] + 1)/poolSize #correct image shape index # construct a fully-connected relu layer layer3 = HiddenLayerRelu(rng, input=layer3_input, n_in=nkerns[2] * imageSize3 * imageSize3, n_out=fullyConnectedNetwork1) layer3.W.set_value(W3,borrow=True) layer3.b.set_value(b3,borrow=True) layer4_input = layer3.out() # construct a fully-connected relu layer layer4 = HiddenLayerRelu(rng, input=layer4_input, n_in=fullyConnectedNetwork1, n_out=fullyConnectedNetwork2) layer4.W.set_value(W4,borrow=True) layer4.b.set_value(b4,borrow=True) # classify the values of the fully-connected sigmoidal layer layer5 = LogisticRegression(input=layer4.output, n_in=fullyConnectedNetwork2, n_out=numberClasses) layer5.W.set_value(W5,borrow=True) layer5.b.set_value(b5,borrow=True) # the cost we minimize during training is the NLL of the model cost = layer5.negative_log_likelihood(y) datasets = load_standard_data() train_set_x, train_set_y = datasets[0] valid_set_x, valid_set_y = datasets[1] test_set_x, test_set_y = datasets[2] #print test_set_y.eval().shape n_test_batches = test_set_y.eval().shape[0] n_valid_batches = valid_set_y.eval().shape[0] n_train_batches = train_set_y.eval().shape[0] # function for guessing the expressions of the test set output_test = theano.function([index], layer5.out(), givens={ x: test_set_x[index * batch_size: (index + 1) * batch_size] }) # create a function to compute the mistakes on the training trained_model = theano.function([index], layer5.errors(y), givens={ x: train_set_x[index * batch_size: (index + 1) * batch_size], y: train_set_y[index * batch_size: (index + 1) * batch_size]}) # create a function to compute the mistakes that are made by the model test_model = theano.function([index], layer5.errors(y), givens={ x: test_set_x[index * batch_size: (index + 1) * batch_size], y: test_set_y[index * batch_size: (index + 1) * batch_size]}) # create a function to compute the mistakes that are made by the model valid_model = theano.function([index], layer5.errors(y), givens={ x: valid_set_x[index * batch_size: (index + 1) * batch_size], y: valid_set_y[index * batch_size: (index + 1) * batch_size]}) # function for guessing the expressions of the test set output_test = theano.function([index], layer5.out(), givens={ x: test_set_x[index * batch_size: (index + 1) * batch_size] }) train_losses = [trained_model(i) for i in xrange(n_train_batches)] train_score = numpy.mean(train_losses) print((' train error rate of model for %d train set: %f %%') % (n_train_batches, train_score * 100.)) # test it on the test set test_losses = [test_model(i) for i in xrange(n_test_batches)] test_score = numpy.mean(test_losses) print((' test error rate of model for %d testcases: %f %%') % (n_test_batches, test_score * 100.)) valid_losses = [valid_model(i) for i in xrange(n_valid_batches)] valid_score = numpy.mean(valid_losses) print((' valid error rate of model for %d valid set: %f %%') % (n_valid_batches, valid_score * 100.)) #for index in range(n_test_batches): # print 'index ', index, 'guess ', output_test(index),'true value ', test_set_y[index * batch_size: (index + 1) * batch_size].eval() confusionMatrix = numpy.zeros((numberClasses,numberClasses)) #print test_set_y[1].eval() for i in range(n_test_batches): #print test_set_y[i].eval(), output_test(i)[0] confusionMatrix[int(test_set_y[i].eval())][int(output_test(i)[0])] = confusionMatrix[int(test_set_y[i].eval())][int(output_test(i)[0])] + 1 #print confusionMatrix[test_set_y[i].eval()][output_test(i)[0]] print confusionMatrix #for index in range(20): # print output_test(index), ' output guess for picture number: ', index # visualize(test_set_x[index * batch_size: (index + 1) * batch_size].eval()) # # confusionMatrix + number of examples per class confusionMatrixPlus = numpy.zeros((numberClasses,numberClasses+1)) casesPerClass = numpy.sum(confusionMatrix, axis=1) confusionMatrixPlus[:,:-1] = confusionMatrix confusionMatrixPlus[:,-1] = casesPerClass print confusionMatrixPlus OA = numpy.trace(confusionMatrix)/n_test_batches #print OA OA_error = 1.0 - OA print OA_error, 'unbalanced error' BA = 0.0 for i in range(numberClasses): BA = confusionMatrixPlus[i,i]/confusionMatrixPlus[i,-1] + BA BA = BA/numberClasses BA_error = 1.0 - BA print BA_error, 'balanced error' print filename return BA_error, OA_error, train_score, valid_score
upLayer2.W.set_value(W2,borrow=True) upLayer2.b.set_value(b2,borrow=True) #fakeLayer2 = HiddenLayerFake(rng, input=upLayer2_input) # classify the values of the fully-connected sigmoidal layer # change number of outputs for different testsets #TODO: n_in = 500 upLayer3 = LogisticRegression(input=upLayer2.out(), n_in=fullyConnectedNetwork, n_out=7) upLayer3.W.set_value(W3,borrow=True) upLayer3.b.set_value(b3,borrow=True) downLayer3 = LogisticRegression(input=upLayer3.out(), n_in=7, n_out=fullyConnectedNetwork) # ##TODO: transpose or inverse ##W3Down = numpy.linalg.pinv(W3) W3Down = numpy.transpose(W3) # W3Down = W3Down[::-1,::-1] # # downLayer3.W.set_value(W3Down,borrow=True) downLayer3.b.set_value(b3,borrow=True) # # ## construct a fully-connected sigmoidal layer downLayer2 = HiddenLayer(rng, input=upLayer2.out(), n_in=nkerns[1] * imageSize2 * imageSize2, n_out=fullyConnectedNetwork)