def trainClassifier(n_classes,lamda,images,labels,options): """ 函数功能:训练softmax训练器 """ # 对数据的维度进行重构以适应softmax训练器的接口 n_train_images = images.shape[1] softmax_images = np.transpose(images, axes = [0,2,3,1]) softmax_images = softmax_images.reshape((int(softmax_images.size / n_train_images), n_train_images)) input_size = int(softmax_images.size / n_train_images) softmax_labels = labels.flatten() - 1 # 确保标注空间的值域为[0,n_classes-1] # 训练softmax分类器 softmax_module = softmax.softmax_train(input_size,n_classes,lamda,softmax_images,softmax_labels,options) #(softmax_theta,softmax_input_size,softmax_n_classes) = softmax_module return softmax_module
pooled_features_train = pooled_features_data['pooled_features_train'] pooled_features_test = pooled_features_data['pooled_features_test'] # Setup parameters for softmax softmax_lambda = 1e-4 n_classes = 4 softmax_input_size = int(pooled_features_train.size / n_train_images) # Reshape the pooled_features to form an input vector for softmax softmax_X = np.transpose(pooled_features_train, axes=[0, 2, 3, 1]) softmax_X = softmax_X.reshape((softmax_input_size, n_train_images)) softmax_Y = train_labels options = {'maxiter': 200, 'disp': True} softmax_model = softmax_train(softmax_input_size, n_classes, softmax_lambda, softmax_X, softmax_Y, options) """ STEP 5: Test classifer Now you will test your trained classifer against the test images """ softmax_input_size = int(pooled_features_test.size / n_test_images) softmax_X = np.transpose(pooled_features_test, axes=[0, 2, 3, 1]) softmax_X = softmax_X.reshape((softmax_input_size, n_test_images)) softmax_Y = test_labels # Make predictions pred = softmax_predict(softmax_model, softmax_X)
print result # ====================================================================== # STEP 4: Train the softmax classifier # This trains the sparse autoencoder on the second autoencoder features. # If you've correctly implemented softmaxCost.m, you don't need # to change anything here. sae2_features = utils_hw.sparse_autoencoder(sae2_opt_theta, hidden_size_L2, hidden_size_L1, sae1_features) options_ = {'maxiter': 400, 'disp': True} softmax_theta, softmax_input_size, softmax_num_classes = softmax.softmax_train(hidden_size_L2, num_classes, lambda_, sae2_features, train_labels, options_) # ====================================================================== # STEP 5: Finetune softmax model # Implement the stacked_autoencoder_cost to give the combined cost of the whole model # then run this cell. # Initialize the stack using the parameters learned stack = [dict() for i in range(2)] stack[0]['w'] = sae1_opt_theta[0:hidden_size_L1 * input_size].reshape(hidden_size_L1, input_size) stack[0]['b'] = sae1_opt_theta[2 * hidden_size_L1 * input_size:2 * hidden_size_L1 * input_size + hidden_size_L1] stack[1]['w'] = sae2_opt_theta[0:hidden_size_L1 * hidden_size_L2].reshape(hidden_size_L2, hidden_size_L1) stack[1]['b'] = sae2_opt_theta[2 * hidden_size_L1 * hidden_size_L2:2 * hidden_size_L1 * hidden_size_L2 + hidden_size_L2]
This trains the sparse autoencoder on the second autoencoder features. If you've correctly implemented softmax_cost, you don't need to change anything here. """ sae2_features = feedforward_autoencoder(sae2_opt_theta, hidden_size_L2, hidden_size_L1, sae1_features) # Instructions: Train the softmax classifier, the classifier takes in # input of dimension "hidden_sizeL2" corresponding to the # hidden layer size of the 2nd layer. # # You should store the optimal parameters in sae_softmax_opt_theta options = {'maxiter': maxiter, 'disp': True} softmax_model = softmax_train(hidden_size_L2, n_classes, lambda_, sae2_features, train_labels, options) softmax_opt_theta = softmax_model['opt_theta'] """ STEP 5: Finetune softmax model """ # Implement the stacked_ae_cost to give the combined cost of the whole model then run this cell. # Initialize the stack using the parameters learned n_stack = 2 # Two layers stack = [{} for i in range(n_stack)] stack[0]['w'] = sae1_opt_theta[0:hidden_size_L1*input_size].reshape((hidden_size_L1, input_size))
W1 = opt_theta[0:hidden_size * input_size].reshape(hidden_size, input_size).transpose() display_network.display_network(W1) ##====================================================================== ## STEP 3: Extract Features from the Supervised Dataset # # You need to complete the code in feedForwardAutoencoder.m so that the # following command will extract features from the data. train_features = sparse_autoencoder.sparse_autoencoder(opt_theta, hidden_size, input_size, train_data) test_features = sparse_autoencoder.sparse_autoencoder(opt_theta, hidden_size, input_size, test_data) ##====================================================================== ## STEP 4: Train the softmax classifier lambda_ = 1e-4 options_ = {'maxiter': 400, 'disp': True} opt_theta, input_size, num_classes = softmax.softmax_train(hidden_size, num_labels, lambda_, train_features, train_labels, options_) ##====================================================================== ## STEP 5: Testing predictions = softmax.softmax_predict((opt_theta, input_size, num_classes), test_features) print "Accuracy: {0:.2f}%".format(100 * np.sum(predictions == test_labels, dtype=np.float64) / test_labels.shape[0])
m=10, factr=10.0, pgtol=1e-8, iprint=1) print 'will calculate l3 features...' l3_activations = feedforward_autoencoder(l3_model, hidden_size, hidden_size, l2_activations) assert l3_activations.shape == (hidden_size, l2_activations.shape[1]) np.save('l3.0to9.model', l3_model) print 'will train classifier...' # train softmax classifier on autoencoded features classifier = softmax.softmax_train(hidden_size, num_classes, softmax_weight_decay, l3_activations, train_labels, max_iter) np.save('softmax.0to9.model', classifier) # use model to predict print 'will load test data...' test_patches, test_labels = get_data('../data/mnist.pkl.gz', train=False, num_samples=num_samples) print 'will compute test features...' test_l2_activations = feedforward_autoencoder(l2_model, hidden_size, visible_size, test_patches) assert test_l2_activations.shape == (hidden_size, test_patches.shape[1])
diff = np.linalg.norm(numgrad - grad) / np.linalg.norm(numgrad + grad) print("Norm of difference = ", diff) # The difference should be small. # In our implementation, these values are usually less than 1e-7. # When your gradients are correct, congratulations! """ STEP 4: Learning parameters Once you have verified that your gradients are correct, you can start training your softmax regression code using softmax_train. """ options = {'maxiter': 100, 'disp': True} model = softmax_train(input_size, n_classes, lambda_, input_data, labels, options) # Although we only use 100 iterations here to train a classifier for the # MNIST data set, in practice, training for more iterations is usually # beneficial. """ STEP 5: Testing You should now test your model against the test images. To do this, you will first need to write softmax_predict, which should return predictions given a softmax model and the input data. """ images = load_MNIST_images('data/mnist/t10k-images-idx3-ubyte') labels = load_MNIST_labels('data/mnist/t10k-labels-idx1-ubyte') input_data = images
""" STEP 4: Train the softmax classifier Use softmax_train from the previous exercise to train a multi-class classifier. Use lambda = 1e-4 for the weight regularization for softmax You need to compute softmax_model using softmax_train on train_features and train_labels """ lambda_ = 1e-4 # weight decay parameter options = {'maxiter': maxiter, 'disp': True} softmax_model = softmax_train(hidden_size, n_labels, lambda_, train_features, train_labels, options) """ STEP 5: Testing Compute Predictions on the test set (test_features) using softmax_predict and softmax_model """ # Make predictions pred = softmax_predict(softmax_model, test_features) acc = np.mean(test_labels == pred) print("The Accuracy (with learned features): {:5.2f}% \n".format(acc*100)) """ Accuracy is the proportion of correctly classified images
lambda l: l <= 4, train=True, num_samples=100000) print 'will calculate training features...' train_activations = feedforward_autoencoder(feature_model, hidden_size, visible_size, train_patches) assert train_activations.shape == (hidden_size, train_patches.shape[1]) print 'will train classifier...' # train softmax classifier on autoencoded features trained = softmax.softmax_train(hidden_size, num_classes, weight_decay, train_activations, train_labels, max_iter) np.save('softmax.0to4.model', trained) # use model to predict print 'will load test data...' test_patches, test_labels = get_data('../data/mnist.pkl.gz', lambda l: l <= 4, train=False, num_samples=100000) print 'will compute test features...' test_activations = feedforward_autoencoder(feature_model, hidden_size,
test_features = feedforward_autoencoder(opt_theta, hidden_size, input_size, test_data) """ STEP 4: Train the softmax classifier Use softmax_train from the previous exercise to train a multi-class classifier. Use lambda = 1e-4 for the weight regularization for softmax You need to compute softmax_model using softmax_train on train_features and train_labels """ lambda_ = 1e-4 # weight decay parameter options = {'maxiter': maxiter, 'disp': True} softmax_model = softmax_train(hidden_size, n_labels, lambda_, train_features, train_labels, options) """ STEP 5: Testing Compute Predictions on the test set (test_features) using softmax_predict and softmax_model """ # Make predictions pred = softmax_predict(softmax_model, test_features) acc = np.mean(test_labels == pred) print("The Accuracy (with learned features): {:5.2f}% \n".format(acc * 100)) """ Accuracy is the proportion of correctly classified images The results for our implementation was:
# Use this to visually compare the gradients side by side print num_grad, grad # Compare numerically computed gradients with the ones obtained from backpropagation diff = np.linalg.norm(num_grad - grad) / np.linalg.norm(num_grad + grad) print diff print "Norm of the difference between numerical and analytical num_grad (should be < 1e-7)\n\n" ##====================================================================== ## STEP 4: Learning parameters # # Once you have verified that your gradients are correct, # you can start training your softmax regression code using softmaxTrain # (which uses minFunc). options_ = {'maxiter': 100, 'disp': True} opt_theta, input_size, num_classes = softmax.softmax_train(input_size, num_classes, lambda_, input_data, labels, options_) ##====================================================================== ## STEP 5: Testing # # You should now test your model against the test images. # To do this, you will first need to write softmaxPredict # (in softmaxPredict.m), which should return predictions # given a softmax model and the input data. test_images = load_MNIST.load_MNIST_images('data/mnist/t10k-images.idx3-ubyte') test_labels = load_MNIST.load_MNIST_labels('data/mnist/t10k-labels.idx1-ubyte') predictions = softmax.softmax_predict((opt_theta, input_size, num_classes), test_images) print "Accuracy: {0:.2f}%".format(100 * np.sum(predictions == test_labels, dtype=np.float64) / test_labels.shape[0])
X_train, y_train = load_training() print "Done" #============================================================= # Step 2: 1-Of-K coding scheme print "One hot encoding..." T_train = one_hot_encoding(y_train) del y_train print "Done" #============================================================= # Step 3: Training softmax start = time.clock() print "Training..." W_optima = softmax_train(X_train, T_train, 2) print "Done" print u"Training stage takes :%.4f seconds" % (time.clock() - start) #============================================================= # Step 4: Testing del X_train, T_train print "Testing..." X_test, y_test = load_testing() Phi_test = add_dummy_variable(X_test) del X_test m = np.shape(Phi_test)[1] k = len(np.unique(y_test)) y_predict = softmax_test(W_optima, Phi_test, m, k) print "Done"
# Load pooled features with open('cnn_pooled_features.pickle', 'r') as f: pooled_features_train = pickle.load(f) pooled_features_test = pickle.load(f) # Setup parameters for softmax softmax_lambda = 1e-4 num_classes = 4 # Reshape the pooled_features to form an input vector for softmax softmax_images = np.transpose(pooled_features_train, axes=[0, 2, 3, 1]) softmax_images = softmax_images.reshape((softmax_images.size / num_train_images, num_train_images)) softmax_labels = train_labels.flatten() - 1 # Ensure that labels are from 0..n-1 (for n classes) options_ = {'maxiter': 1000, 'disp': True} softmax_model = softmax.softmax_train(softmax_images.size / num_train_images, num_classes, softmax_lambda, softmax_images, softmax_labels, options_) (softmax_opt_theta, softmax_input_size, softmax_num_classes) = softmax_model ##====================================================================== ## STEP 5: Test classifer # Now you will test your trained classifer against the test images softmax_images = np.transpose(pooled_features_test, axes=[0, 2, 3, 1]) softmax_images = softmax_images.reshape((softmax_images.size / num_test_images, num_test_images)) softmax_labels = test_labels.flatten() - 1 predictions = softmax.softmax_predict(softmax_model, softmax_images) print "Accuracy: {0:.2f}%".format(100 * np.sum(predictions == softmax_labels, dtype=np.float64) / test_labels.shape[0]) # You should expect to get an accuracy of around 80% on the test images.
print("Norm of difference = ", diff) # The difference should be small. # In our implementation, these values are usually less than 1e-7. # When your gradients are correct, congratulations! """ STEP 4: Learning parameters Once you have verified that your gradients are correct, you can start training your softmax regression code using softmax_train. """ options = {'maxiter': 100, 'disp': True} model = softmax_train(input_size, n_classes, lambda_, input_data, labels, options) # Although we only use 100 iterations here to train a classifier for the # MNIST data set, in practice, training for more iterations is usually # beneficial. """ STEP 5: Testing You should now test your model against the test images. To do this, you will first need to write softmax_predict, which should return predictions given a softmax model and the input data. """ images = load_MNIST_images('data/mnist/t10k-images-idx3-ubyte') labels = load_MNIST_labels('data/mnist/t10k-labels-idx1-ubyte') input_data = images
print 'will calculate l3 features...' l3_activations = feedforward_autoencoder(l3_model, hidden_size, hidden_size, l2_activations) assert l3_activations.shape == (hidden_size, l2_activations.shape[1]) np.save('l3.0to9.model', l3_model) print 'will train classifier...' # train softmax classifier on autoencoded features classifier = softmax.softmax_train(hidden_size, num_classes, softmax_weight_decay, l3_activations, train_labels, max_iter) np.save('softmax.0to9.model', classifier) # use model to predict print 'will load test data...' test_patches, test_labels = get_data('../data/mnist.pkl.gz',
pooled_features_train = pooled_features_data['pooled_features_train'] pooled_features_test = pooled_features_data['pooled_features_test'] # Setup parameters for softmax softmax_lambda = 1e-4 n_classes = 4 softmax_input_size = int(pooled_features_train.size / n_train_images) # Reshape the pooled_features to form an input vector for softmax softmax_X = np.transpose(pooled_features_train, axes=[0, 2, 3, 1]) softmax_X = softmax_X.reshape((softmax_input_size, n_train_images)) softmax_Y = train_labels options = {'maxiter': 200, 'disp': True} softmax_model = softmax_train(softmax_input_size, n_classes, softmax_lambda, softmax_X, softmax_Y, options) """ STEP 5: Test classifer Now you will test your trained classifer against the test images """ softmax_input_size = int(pooled_features_test.size / n_test_images) softmax_X = np.transpose(pooled_features_test, axes=[0, 2, 3, 1]) softmax_X = softmax_X.reshape((softmax_input_size, n_test_images)) softmax_Y = test_labels # Make predictions pred = softmax_predict(softmax_model, softmax_X) acc = np.mean(softmax_Y == pred)