def main(): # getting the subset dataset from MNIST # binary classification for digits 1 and 7 digit_range = [1, 7] data, label, test_data, test_label = \ mnist(noTrSamples=2400,noTsSamples=1000,\ digit_range=digit_range,\ noTrPerClass=1200, noTsPerClass=500) temp1, temp2, val_data, val_label = mnist(noTrSamples=2, noTsSamples=400, digit_range=digit_range, noTrPerClass=1, noTsPerClass=200) train_data = np.concatenate([data[:, :1000], data[:, 1200:2200]], axis=1) val_data = np.concatenate([data[:, 1000:1200], data[:, 2200:2400]], axis=1) train_label = np.concatenate([label[:, :1000], label[:, 1200:2200]], axis=1) val_label = np.concatenate([label[:, 1000:1200], label[:, 2200:2400]], axis=1) #convert to binary labels train_label[train_label == digit_range[0]] = 0 train_label[train_label == digit_range[1]] = 1 test_label[test_label == digit_range[0]] = 0 test_label[test_label == digit_range[1]] = 1 val_label[val_label == digit_range[0]] = 0 val_label[val_label == digit_range[1]] = 1 n_in, m = train_data.shape n_fin = 1 n_h = 500 net_dims = [n_in, n_h, n_fin] # initialize learning rate and num_iterations learning_rate = 0.01 num_iterations = 500 costs_tr, costs_val, parameters_tr = two_layer_network(train_data, train_label, val_data, val_label, net_dims, \ num_iterations=num_iterations, learning_rate=learning_rate) # compute the accuracy for training set and testing set train_Pred = classify(train_data, parameters_tr) test_Pred = classify(test_data, parameters_tr) trAcc = accuracy(train_Pred, train_label) teAcc = accuracy(test_Pred, test_label) print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc)) # CODE HERE TO PLOT costs vs iterations plt.xlabel("iterations") plt.ylabel("cost") plt.title("BINARY - iterations vs cost for train and validation data") plt.plot(costs_tr, "-g", label='train') plt.plot(costs_val, ":b", label='val') plt.legend() plt.show()
def main(): # load the data from external load_mnist.py script train_X, train_y, test_X, test_y = mnist(noTrSamples=400, noTsSamples=100, digit_range=[5, 8], noTrPerClass=200, noTsPerClass=50) # get pca data pca_train_X, e_pca_train_X = pca(train_X, dim=10) pca_test_X, e_pca_test_X = pca(test_X, dim=10) # apply Fishers Linear Discriminant to project PCA trained data pca_mnist_fld = FLD(pca_train_X, train_y, dim=1) # fit the FLD process pca_mnist_fld.fit() # compute training accuracy train_acc = pca_mnist_fld.train_accuracy() # compute test accuracy test_acc = pca_mnist_fld.test_accuracy(pca_test_X, test_y) # Display training and test accuracy print(f'Training accuracy : {train_acc}') print(f'Test accuracy : {test_acc}') clf = LinearDiscriminantAnalysis() # print(pca_train_X.T.shape, train_y.T.flatten().shape) # (400, 10) (400,) clf.fit(pca_train_X.T, train_y.T.flatten()) print('From sklearn') print( 'Training accuracy : ', clf.fit(pca_train_X.T, train_y.T.flatten()).score(pca_train_X.T, train_y.T.flatten())) print('Test accuracy : ', clf.score(pca_test_X.T, test_y.T.flatten()))
def main(): digit_range = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] X_train, _, X_test , _ = \ mnist(noTrSamples=6000, noTsSamples=1000, \ digit_range=digit_range, \ noTrPerClass=600, noTsPerClass=100) X_noisytrain = noisyMethod1(X_train) # X_noisytrain = noisyMethod2(X_train, 10) # X_noisytrain = X_noisytrain.T # X_train = X_train.T num_epochs = 1000 learning_rate = 0.01 batch_size = 256 net_dims = [784, 1000, 784] parameters, costs = encoder_decoder_twolayer(X_noisytrain, X_train, net_dims, num_epochs, batch_size, learning_rate) l1, _ = sigmoid(np.dot(parameters["W1"], X_train) + parameters["b1"]) Image, _ = sigmoid(np.dot(parameters["W2"], l1) + parameters["b2"]) fig, axes = plt.subplots(3, 5) for i in range(5): axes[0, i].imshow(X_train[:, i].reshape((28, 28)), cmap="gray") axes[1, i].imshow(X_noisytrain[:, i].reshape((28, 28)), cmap='gray') axes[2, i].imshow(Image[:, i].reshape((28, 28)), cmap="gray") plt.savefig("Original_Noisy_Reconstructed_Images.png") plt.show()
def main(): # getting the subset dataset from MNIST # binary classification for digits 1 and 7 digit_range = [1, 7] train_data, train_label, test_data, test_label = \ mnist(noTrSamples=2400,noTsSamples=1000,\ digit_range=digit_range,\ noTrPerClass=1200, noTsPerClass=500) #convert to binary labels train_label[train_label == digit_range[0]] = 0 train_label[train_label == digit_range[1]] = 1 test_label[test_label == digit_range[0]] = 0 test_label[test_label == digit_range[1]] = 1 n_in, m = train_data.shape n_fin = 1 n_h = 500 net_dims = [n_in, n_h, n_fin] # initialize learning rate and num_iterations learning_rate = 0.1 num_iterations = 1000 costs, parameters = two_layer_network(train_data, train_label, net_dims, \ num_iterations=num_iterations, learning_rate=learning_rate) # compute the accuracy for training set and testing set train_Pred = classify(train_data, parameters) test_Pred = classify(test_data, parameters) trAcc = (np.sum(train_Pred == train_label) / train_data.shape[1]) * 100 teAcc = (np.sum(test_Pred == test_label) / test_data.shape[1]) * 100 print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
def main(): n_in, m = 784,784 n_h1 = 500 n_h2 = 100 net_dims = [n_in, n_h1, n_h2] #net_dims = ast.literal_eval( sys.argv[1] ) net_dims.append(10) # Adding the digits layer with dimensionality = 10 print("Network dimensions are:" + str(net_dims)) # getting the subset dataset from MNIST train_data, train_label, test_data, test_label = \ mnist(noTrSamples=6000,noTsSamples=1000,\ digit_range=[0,1,2,3,4,5,6,7,8,9],\ noTrPerClass=600, noTsPerClass=100) learning_rate_array = [0.0007, 0.0001, 0.00007] num_iterations = 300 batch_size = len(train_label) plots = [] for learning_rate in learning_rate_array: costs, val_loss, parameters = multi_layer_network(train_data, train_label, net_dims, num_iterations=num_iterations, learning_rate=learning_rate, batch_size=batch_size) # compute the accuracy for training set and testing set train_Pred = classify(train_data, parameters) test_Pred = classify(test_data, parameters) test_error = ((test_Pred[0]!=test_label[0]).sum())/ test_Pred[0].sum() print("Test error: {0:0.4f} ".format(test_error)) trAcc = train_Pred - train_label teAcc = test_Pred - test_label trAcc[trAcc != 0] = 1 teAcc[teAcc != 0] = 1 trAcc = ( 1 - np.count_nonzero(train_Pred - train_label ) / float(train_Pred.shape[1])) * 100 teAcc = ( 1 - np.count_nonzero(test_Pred - test_label ) / float(test_Pred.shape[1]) ) * 100 print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc)) # points = np.arange(0, 300) # plot_train, = plt.plot(costs[:len(costs)]) # # plots.append(plot_train) plot_val, = plt.plot(val_loss) plots.append(plot_val) plt.xlabel("No Of Iterations") plt.ylabel("Cost/ val_loss") plt.title("Loss vs Number of Iterations with varying Learning rates for ADAM") plt.legend(plots,learning_rate_array) plt.savefig("Loss plot for ADAM's Momentum")
def main(): # getting the subset dataset from MNIST # binary classification for digits 1 and 7 validation_costs = [] test_accuracies = [] # getting the subset dataset from MNIST data, data_label, test_data, test_label = \ mnist(noTrSamples=60000,noTsSamples=500,\ digit_range=[0,1,2,3,4,5,6,7,8,9],\ noTrPerClass=6000, noTsPerClass=50) print(test_label) # initialize learning rate and num_iterations print(data_label) num_iterations = 1000 n_in, m = data.shape n_fin = 784 #n_h = [900,1000,1100,1200,2000] n_h = [1000] count = 1 for h in n_h: print("Hidden layer", h) net_dims = [n_in, h, n_fin] # initialize learning rate and num_iterations learning_rate = 0.1 #fig = plt.figure(figsize=(15, 10)) costs, parameters = two_layer_network(data, data_label,net_dims, \ num_iterations=num_iterations, learning_rate=learning_rate) sigma = [0.1, 0.5, 1, 5, 10, 15] costs = [] for sig in sigma: YPred = classify(noise_induce(test_data, sig), parameters) cost_pred = reconstruction_loss(YPred, test_data) print("Cost prediction", cost_pred) count = 1 costs.append(cost_pred) for s in range(10): ind = np.argwhere(test_label[0] == s)[0][0] print(ind, test_label[0][ind]) img = (YPred.T[ind]).reshape(28, 28) plt.subplot(5, 2, count) count += 1 plt.xticks([]) plt.yticks([]) plt.grid(False) plt.imshow(img, cmap=plt.cm.binary) plt.show() print(costs)
def main(): ''' Trains a multilayer network for MNIST digit classification (all 10 digits) To create a network with 1 hidden layer of dimensions 800 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800]" The network will have the dimensions [784,800,10] 784 is the input size of digit images (28pix x 28pix = 784) 10 is the number of digits To create a network with 2 hidden layers of dimensions 800 and 500 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800,500]" The network will have the dimensions [784,800,500,10] 784 is the input size of digit images (28pix x 28pix = 784) 10 is the number of digits ''' # net_dims = ast.literal_eval(sys.argv[1]) net_dims = [784,800,500,300] net_dims.append(10) # Adding the digits layer with dimensionality = 10 print("Network dimensions are:" + str(net_dims)) # getting the subset dataset from MNIST train_data, train_label, test_data, test_label = \ mnist(noTrSamples=6000,noTsSamples=1000,digit_range=[0,10],\ noTrPerClass=600, noTsPerClass=100) # initialize learning rate and num_iterations learning_rate = 0.2 num_iterations = 500 costs, parameters = multi_layer_network(train_data, train_label, net_dims, \ num_iterations=num_iterations, learning_rate=learning_rate) # compute the accuracy for training set and testing set train_Pred = classify(train_data, parameters) test_Pred = classify(test_data, parameters) trAcc = (1 - np.count_nonzero(train_Pred - train_label) / float(train_Pred.shape[1])) * 100 teAcc = (1 - np.count_nonzero(test_Pred - test_label) / float(test_Pred.shape[1])) * 100 print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc)) ### CODE HERE to plot costs X = range(0, num_iterations, 10) plt.plot(X, costs) plt.xlabel("Iterations") plt.ylabel("Cost") plt.show()
def main(): # load the data from external load_mnist.py script train_X, train_y, test_X, test_y = mnist(noTrSamples=400, noTsSamples=100, digit_range=[5, 8], noTrPerClass=200, noTsPerClass=50) # perform PCA on test and train data # Also get the eigen vectors for reconstruction purpose pca_train_X, e_pca_train_X = pca(train_X, dim=10) pca_test_X, e_pca_test_X = pca(test_X, dim=10) # plot the covariance matrix of PCA transformed train data plot_covariance_matrix(pca_train_X) # reconstruct images from PCA transformed train data and eigen vectors matrix recon_train_X = reconstruct_data(pca_train_X, e_pca_train_X) # show the reconstructed image samples show_original_and_recon(train_X, recon_train_X, no_of_samples=5)
def main(): trainDevX, trainDevY, testX, testY = \ mnist(noTrSamples=50000,noTsSamples=5000,\ digit_range=[0,1,2,3,4,5,6,7,8,9],\ noTrPerClass=5000, noTsPerClass=500) perm = np.random.permutation(trainDevX.shape[1]) trainDevX = trainDevX[:, perm] trainDevY = trainDevY[:, perm] devX = trainDevX[:, 45000:50000] devY = trainDevY[:, 45000:50000] trainX = trainDevX[:, 0:45000] trainY = trainDevY[:, 0:45000] #trainY = one_hot(trainY.astype(int), 10) #devY = one_hot(devY.astype(int), 10) #testY = one_hot(testY.astype(int), 10) lDim = [784, 500, 10] miniBatchSize = 100 k = [[50], [100]] nEpoch = 200 alpha = 0.001 costHL = [] accTest = [] errDev = [] errTest = [] for i in range(len(k)): print(k[i]) parameters, costTrain, costDev, t = neuralNetwork( trainX, trainY, lDim, nEpoch, alpha, devX, devY, k[i], miniBatchSize) costHL.append((costTrain, costDev)) errDev.append(costDev[nEpoch - 1]) Ypred, _ = predictClass(devX, parameters) accDev.append(accuracy(devY, Ypred)) Ypred, AL = predictClass(testX, parameters) accTest.append(accuracy(testY, Ypred)) errTest.append(costNN(AL, one_hot(testY.astype(int), 10)))
def main(): net_dims = ast.literal_eval( sys.argv[1] ) net_dims.append(10) print("Network dimensions are:" + str(net_dims)) train_data, train_label, test_data, test_label = \ mnist(noTrSamples=50000,noTsSamples=5000,\ digit_range=[0,1,2,3,4,5,6,7,8,9],\ noTrPerClass=5000, noTsPerClass=500) learning_rate = 0.001 num_iterations = 500 mini_batch_size = 512 gamma = 0.9 beta = 0.999 NAG_coeff = 0.999 algorithm = ['classical', 'momentum', 'rmsprop', 'adam', 'NAG'] print(algorithm[4]) train_costs, parameters = multi_layer_network(train_data, train_label, net_dims, algorithm[4], num_iterations, learning_rate, gamma, beta, NAG_coeff, mini_batch_size) train_pred = classify(train_data, parameters) test_pred = classify(test_data, parameters) trAcc = (np.count_nonzero(train_pred == train_label) / train_data.shape[1]) * 100 teAcc = (np.count_nonzero(test_pred == test_label) / test_data.shape[1]) * 100 print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc)) print("Cost for training set is {0:0.3f} ".format(train_costs[len(train_costs)-1])) plt.title("training costs; learning rate="+str(learning_rate)) plt.xlabel("iterations") plt.ylabel("cost") plt.plot(list(range(len(train_costs))), train_costs, label='training cost') plt.legend() plt.show()
def main(): net_dims = ast.literal_eval(sys.argv[1]) net_dims.append(10) # Adding the digits layer with dimensionality = 10 print("Network dimensions are:" + str(net_dims)) # getting the subset dataset from MNIST Original_train_data, Original_train_label, test_data, test_label = \ mnist(noTrSamples=60000,noTsSamples=10000,\ digit_range=[0,1,2,3,4,5,6,7,8,9],\ noTrPerClass=6000, noTsPerClass=1000) train_data = Original_train_data[:, :5000] train_label = Original_train_label[:, :5000] validation_data = Original_train_data[:, 5000:6000] validation_label = Original_train_label[:, 5000:6000] for i in range(6000, 60000, 6000): train_data = np.hstack((train_data, Original_train_data[:, i:i + 5000])) train_label = np.hstack( (train_label, Original_train_label[:, i:i + 5000])) validation_data = np.hstack( (validation_data, Original_train_data[:, i + 5000:i + 6000])) validation_label = np.hstack( (validation_label, Original_train_label[:, i + 5000:i + 6000])) print(train_data.shape) print(validation_data.shape) mini_batches_train = [] mini_batches_val = [] for j in range(10): x = get_mini_batches(train_data, train_label, 500) for i in range(len(x)): mini_batches_train.append(x[i]) y = get_mini_batches(validation_data, validation_label, 100) for i in range(len(y)): mini_batches_val.append(y[i]) print(len(mini_batches_train)) print(len(mini_batches_val)) learning_rate = 0.1 num_iterations = 100 num_iter = 1 minibatchsize_train = 500 costs = [] valid_costs = [] parameters = initialize_multilayer_weights(net_dims) for i in range(len(mini_batches_train)): loss, parameters, v_loss = multi_layer_network(mini_batches_train[i][0],mini_batches_train[i][1],mini_batches_val[i][0],mini_batches_val[i][1],parameters,net_dims, \ num_iterations=num_iterations, learning_rate=learning_rate,decay_rate=0.01) #parameters=parameters2.copy() #parameters2.clear() if i % 10 == 0: costs.append(loss) valid_costs.append(v_loss) with open('output_nomomentum.txt', 'w') as f: for item in valid_costs: f.write("%s\n" % item) min_train_loss = min(costs) print("min_train_loss", min_train_loss) train_Pred = classify(train_data, train_label, parameters) test_Pred = classify(test_data, test_label, parameters) valid_Pred = classify(validation_data, validation_label, parameters) trAcc = 100 * (float(np.sum(train_Pred == train_label)) / train_label.shape[1]) teAcc = 100 * (float(np.sum(test_Pred == test_label)) / test_label.shape[1]) VaAcc = 100 * (float(np.sum(valid_Pred == validation_label)) / validation_label.shape[1]) print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc)) print("Accuracy for Validation set is {0:0.3f} %".format(VaAcc)) ### CODE HERE to plot costs plt.plot(costs) #plt.show() plt.plot(valid_costs) plt.ylabel('costs') plt.xlabel('iterations (multiples of 10)') plt.title( "Validation and training costs vs iterations at learning rate 0.2") plt.show() print("Test Error:", 1 - (teAcc / 100))
def main(): start_time = time.time() train_data, train_label, test_data, test_label = mnist() noise = 0.4 testnoise1 = 0.1 testnoise2 = 0.2 testnoise3 = 0.3 testnoise4 = 0.4 trX_noisy = get_corrupted_input(train_data, noise) tsX_noisy1 = get_corrupted_input(test_data, testnoise1) tsX_noisy2 = get_corrupted_input(test_data, testnoise2) tsX_noisy3 = get_corrupted_input(test_data, testnoise3) tsX_noisy4 = get_corrupted_input(test_data, testnoise4) n_in, m = train_data.shape n_fin, m = train_data.shape n_h = 1000 net_dims = [n_in, n_h, n_fin] # initialize learning rate and num_iterations learning_rate = 1 num_iterations = 1000 costs, parameters = two_layer_network(trX_noisy, train_data, net_dims, num_iterations=num_iterations, learning_rate=learning_rate) fig = plt.figure() plt.imshow(test_data[:, 20].reshape(28, -1), cmap=plt.cm.binary) plt.title("Test_Sample with gaussian noise" + str(noise) + "is") plt.show() fig.savefig("Test_Sample with gaussian noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(test_data[:, 80].reshape(28, -1), cmap=plt.cm.binary) plt.title("Test_Sample with gaussian noise" + str(noise) + "is") plt.show() fig.savefig("Test_Sample 8with gaussian noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(test_data[:, 91].reshape(28, -1), cmap=plt.cm.binary) plt.title("Test_Sample with gaussian noise" + str(noise) + "is") plt.show() fig.savefig("Test_Sample 9 with gaussian noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy1[:, 20].reshape(28, -1), cmap=plt.cm.binary) plt.title("Noisy_Test_Sample..with_gaussian_noise" + str(testnoise1) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Samplewith_gaussian_noise" + str(testnoise1) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy2[:, 20].reshape(28, -1), cmap=plt.cm.binary) plt.title("Noisy_Test_Sample withgaussiannoise" + str(testnoise2) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample with_gaussiannoise" + str(testnoise2) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy3[:, 20].reshape(28, -1), cmap=plt.cm.binary) plt.title("Noisy_Test_Sample with_gaussian_noise" + str(testnoise3) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample with_gaussian_noise" + str(testnoise3) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy4[:, 20].reshape(28, -1), cmap=plt.cm.binary) plt.title("Noisy_Test_Sample with_gaussian_noise" + str(testnoise4) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample with_gaussian_noise" + str(testnoise4) + "and train noise" + str(noise) + "is.png") fig = plt.figure() fig = plt.figure() plt.imshow(tsX_noisy1[:, 80].reshape(28, -1), cmap=plt.cm.binary) plt.title("Noisy_Test_Sample 8with_gaussian_noise" + str(testnoise1) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample8with_gaussian_noise" + str(testnoise1) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy2[:, 80].reshape(28, -1), cmap=plt.cm.binary) plt.title("Noisy_Test_Sample 8 with_gaussian_noise" + str(testnoise2) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample8 with_gaussian_noise" + str(testnoise2) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy3[:, 80].reshape(28, -1), cmap=plt.cm.binary) plt.title("Noisy_Test_Sample 8with_gaussian_noise" + str(testnoise3) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample 8with_gaussian_noise" + str(testnoise3) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy4[:, 80].reshape(28, -1), cmap=plt.cm.binary) plt.title("Noisy_Test_Sample 8with_gaussian_noise" + str(testnoise4) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample8 with_gaussian_noise" + str(testnoise4) + "and train noise" + str(noise) + "is.png") fig = plt.figure() fig = plt.figure() plt.imshow(tsX_noisy1[:, 91].reshape(28, -1), cmap=plt.cm.binary) plt.title("Noisy_Test_Sample 9with_gaussian_noise" + str(testnoise1) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample8with_gaussian_noise" + str(testnoise1) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy2[:, 91].reshape(28, -1), cmap=plt.cm.binary) plt.title("Noisy_Test_Sample 9 with_gaussian_noise" + str(testnoise2) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample9 with_gaussian_noise" + str(testnoise2) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy3[:, 91].reshape(28, -1), cmap=plt.cm.binary) plt.title("Noisy_Test_Sample 9with_gaussian_noise" + str(testnoise3) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample 9 with_gaussian_noise" + str(testnoise3) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy4[:, 91].reshape(28, -1), cmap=plt.cm.binary) plt.title("Noisy_Test_Sample 9 with_gaussian_noise" + str(testnoise4) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample 9 with_gaussian_noise" + str(testnoise4) + "and train noise" + str(noise) + "is.png") test_Pred1 = denoise(tsX_noisy1, parameters) test_Pred2 = denoise(tsX_noisy2, parameters) test_Pred3 = denoise(tsX_noisy3, parameters) test_Pred4 = denoise(tsX_noisy4, parameters) print("accuracy of test sample 1 with gaussian noise" + str(testnoise1) + "is " + str((1 - error(test_Pred1, test_data)) * 100) + "%") print("accuracy of test sample 2 with gaussian noise" + str(testnoise2) + "is " + str((1 - error(test_Pred2, test_data)) * 100) + "%") print("accuracy of test sample 3 with gaussian noise" + str(testnoise3) + "is " + str((1 - error(test_Pred3, test_data)) * 100) + "%") print("accuracy of test sample 4 with gaussian noise" + str(testnoise4) + "is " + str((1 - error(test_Pred4, test_data)) * 100) + "%") fig = plt.figure() plt.imshow(test_Pred1[:, 20].reshape(28, -1), cmap=plt.cm.binary) plt.title("denoised_Sample with_gaussian_noise" + str(testnoise1) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random noise_gaussian_Test_Sample op1") fig = plt.figure() plt.imshow(test_Pred2[:, 20].reshape(28, -1), cmap=plt.cm.binary) plt.title("denoised_Sample with_gaussian_noise" + str(testnoise2) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_gaussian noiseTest_Sample op2") fig = plt.figure() plt.imshow(test_Pred3[:, 20].reshape(28, -1), cmap=plt.cm.binary) plt.title("denoised_Sample with_gaussian_noise" + str(testnoise3) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_gaussian_noiseTest_Sample op3") fig = plt.figure() plt.imshow(test_Pred4[:, 20].reshape(28, -1), cmap=plt.cm.binary) plt.title("denoised_Sample with_gaussian_noise" + str(testnoise4) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_gaussian noiseTest_Sample op4") fig = plt.figure() plt.imshow(test_Pred1[:, 80].reshape(28, -1), cmap=plt.cm.binary) plt.title("denoised_Sample 8 with_gaussian_noise" + str(testnoise1) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random noise_gaussian_Test_Sample8 op1") fig = plt.figure() plt.imshow(test_Pred2[:, 80].reshape(28, -1), cmap=plt.cm.binary) plt.title("denoised_Sample with_gaussian_noise" + str(testnoise2) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random noiseTest_Sample 8 op2") fig = plt.figure() plt.imshow(test_Pred3[:, 80].reshape(28, -1), cmap=plt.cm.binary) plt.title("denoised_Sample with_gaussian_noise" + str(testnoise3) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_gaussian_noiseTest_Sample 8 op3") fig = plt.figure() plt.imshow(test_Pred4[:, 80].reshape(28, -1), cmap=plt.cm.binary) plt.title("denoised_Sample with_gaussian_noise" + str(testnoise4) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_gaussian noiseTest_Sample 8 op4") fig = plt.figure() plt.imshow(test_Pred1[:, 91].reshape(28, -1), cmap=plt.cm.binary) plt.title("denoised_Sample 9 with_gaussian_noise" + str(testnoise1) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random noise_gaussian_Test_Sample9 op1") fig = plt.figure() plt.imshow(test_Pred2[:, 91].reshape(28, -1), cmap=plt.cm.binary) plt.title("denoised_Sample 9 with_random_noise" + str(testnoise2) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_gaussian noiseTest_Sample 9 op2") fig = plt.figure() plt.imshow(test_Pred3[:, 91].reshape(28, -1), cmap=plt.cm.binary) plt.title("denoised_Sample 9 with_gaussian_noise" + str(testnoise3) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_gaussian_noiseTest_Sample 9 op3") fig = plt.figure() plt.imshow(test_Pred4[:, 91].reshape(28, -1), cmap=plt.cm.binary) plt.title("denoised_Sample 9 with_gaussian_noise" + str(testnoise4) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_gaussian noiseTest_Sample 9 op4") plt.plot(costs, label='Training cost') plt.title("training cost with learning rate =" + str(learning_rate) + "iterations" + str(num_iterations) + "noise :" + str(noise)) plt.show() fig.savefig("training cost") print("Total execution time: %s minutes!!" % ((time.time() - start_time) // 60))
def main(): start_time = time.time() train_data, train_label, test_data, test_label = mnist() # print(param) n_rows = train_data.shape[0] n_cols = train_data.shape[1] test_rows = test_data.shape[0] test_cols = test_label.shape[1] noise = 5 testnoise1 = 4 testnoise2 = 5 testnoise3 = 6 testnoise4 = 7 trX_noisy = masking_noise(train_data, noise) tsX_noisy1 = masking_noise(test_data, testnoise1) tsX_noisy2 = masking_noise(test_data, testnoise2) tsX_noisy3 = masking_noise(test_data, testnoise3) tsX_noisy4 = masking_noise(test_data, testnoise4) n_in, m = train_data.shape n_fin, m = train_data.shape n_h = 1000 net_dims = [n_in, n_h, n_fin] # initialize learning rate and num_iterations learning_rate = 1 num_iterations = 1000 costs, parameters = two_layer_network(trX_noisy, train_data, net_dims, num_iterations=num_iterations, learning_rate=learning_rate) fig = plt.figure() plt.imshow(test_data[:, 99].reshape(28, -1), cmap="gray") plt.title("Test_Sample with random noise" + str(noise) + "is") plt.show() fig.savefig("Test_Sample with random noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(test_data[:, 72].reshape(28, -1), cmap="gray") plt.title("Test_Sample with random noise" + str(noise) + "is") plt.show() fig.savefig("Test_Sample 8with random noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(test_data[:, 20].reshape(28, -1), cmap="gray") plt.title("Test_Sample with random noise" + str(noise) + "is") plt.show() fig.savefig("Test_Sample 9 with random noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy1[:, 99].reshape(28, -1), cmap="gray") plt.title("Noisy_Test_Sample..with_random_noise" + str(testnoise1) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Samplewith_random_noise" + str(testnoise1) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy2[:, 99].reshape(28, -1), cmap="gray") plt.title("Noisy_Test_Sample with_random_noise" + str(testnoise2) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample with_random_noise" + str(testnoise2) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy3[:, 99].reshape(28, -1), cmap="gray") plt.title("Noisy_Test_Sample with_random_noise" + str(testnoise3) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample with_random_noise" + str(testnoise3) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy4[:, 99].reshape(28, -1), cmap="gray") plt.title("Noisy_Test_Sample with_random_noise" + str(testnoise4) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample with_random_noise" + str(testnoise4) + "and train noise" + str(noise) + "is.png") fig = plt.figure() fig = plt.figure() plt.imshow(tsX_noisy1[:, 72].reshape(28, -1), cmap="gray") plt.title("Noisy_Test_Sample 8with_random_noise" + str(testnoise1) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample8with_random_noise" + str(testnoise1) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy2[:, 72].reshape(28, -1), cmap="gray") plt.title("Noisy_Test_Sample 8 with_random_noise" + str(testnoise2) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample8 with_random_noise" + str(testnoise2) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy3[:, 72].reshape(28, -1), cmap="gray") plt.title("Noisy_Test_Sample 8with_random_noise" + str(testnoise3) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample 8with_random_noise" + str(testnoise3) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy4[:, 72].reshape(28, -1), cmap="gray") plt.title("Noisy_Test_Sample 8with_random_noise" + str(testnoise4) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample8 with_random_noise" + str(testnoise4) + "and train noise" + str(noise) + "is.png") fig = plt.figure() fig = plt.figure() plt.imshow(tsX_noisy1[:, 20].reshape(28, -1), cmap="gray") plt.title("Noisy_Test_Sample 9with_random_noise" + str(testnoise1) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample8with_random_noise" + str(testnoise1) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy2[:, 20].reshape(28, -1), cmap="gray") plt.title("Noisy_Test_Sample 9 with_random_noise" + str(testnoise2) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample9 with_random_noise" + str(testnoise2) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy3[:, 20].reshape(28, -1), cmap="gray") plt.title("Noisy_Test_Sample 9with_random_noise" + str(testnoise3) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample 9 with_random_noise" + str(testnoise3) + "and train noise" + str(noise) + "is.png") fig = plt.figure() plt.imshow(tsX_noisy4[:, 20].reshape(28, -1), cmap="gray") plt.title("Noisy_Test_Sample 9 with_random_noise" + str(testnoise4) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Noisy_Test_Sample 9 with_random_noise" + str(testnoise4) + "and train noise" + str(noise) + "is.png") test_Pred1 = denoise(tsX_noisy1, parameters) test_Pred2 = denoise(tsX_noisy2, parameters) test_Pred3 = denoise(tsX_noisy3, parameters) test_Pred4 = denoise(tsX_noisy4, parameters) print("accuracy of test sample 2 with random noise" + str(testnoise1) + "is " + str((1 - error(test_Pred1, test_data)) * 100) + "%") print("accuracy of test sample 2 with random noise" + str(testnoise2) + "is " + str((1 - error(test_Pred2, test_data)) * 100) + "%") print("accuracy of test sample 2 with random noise" + str(testnoise3) + "is " + str((1 - error(test_Pred3, test_data)) * 100) + "%") print("accuracy of test sample 2 with random noise" + str(testnoise4) + "is " + str((1 - error(test_Pred4, test_data)) * 100) + "%") fig = plt.figure() plt.imshow(test_Pred1[:, 99].reshape(28, -1), cmap="gray") plt.title("denoised_Sample with_random_noise" + str(testnoise1) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random noise_random_Test_Sample op1") fig = plt.figure() plt.imshow(test_Pred2[:, 99].reshape(28, -1), cmap="gray") plt.title("denoised_Sample with_random_noise" + str(testnoise2) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random noiseTest_Sample op2") fig = plt.figure() plt.imshow(test_Pred3[:, 99].reshape(28, -1), cmap="gray") plt.title("denoised_Sample with_random_noise" + str(testnoise3) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random_noiseTest_Sample op3") fig = plt.figure() plt.imshow(test_Pred4[:, 99].reshape(28, -1), cmap="gray") plt.title("denoised_Sample with_random_noise" + str(testnoise4) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random noiseTest_Sample op4") fig = plt.figure() plt.imshow(test_Pred1[:, 72].reshape(28, -1), cmap="gray") plt.title("denoised_Sample 8 with_random_noise" + str(testnoise1) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random noise_random_Test_Sample8 op1") fig = plt.figure() plt.imshow(test_Pred2[:, 72].reshape(28, -1), cmap="gray") plt.title("denoised_Sample with_random_noise" + str(testnoise2) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random noiseTest_Sample 8 op2") fig = plt.figure() plt.imshow(test_Pred3[:, 72].reshape(28, -1), cmap="gray") plt.title("denoised_Sample with_random_noise" + str(testnoise3) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random_noiseTest_Sample 8 op3") fig = plt.figure() plt.imshow(test_Pred4[:, 72].reshape(28, -1), cmap="gray") plt.title("denoised_Sample with_random_noise" + str(testnoise4) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random noiseTest_Sample 8 op4") fig = plt.figure() plt.imshow(test_Pred1[:, 20].reshape(28, -1), "gray") plt.title("denoised_Sample 9 with_random_noise" + str(testnoise1) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random noise_random_Test_Sample9 op1") fig = plt.figure() plt.imshow(test_Pred2[:, 20].reshape(28, -1), cmap="gray") plt.title("denoised_Sample 9 with_random_noise" + str(testnoise2) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random noiseTest_Sample 9 op2") fig = plt.figure() plt.imshow(test_Pred3[:, 20].reshape(28, -1), cmap="gray") plt.title("denoised_Sample 9 with_random_noise" + str(testnoise3) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random_noiseTest_Sample 9 op3") fig = plt.figure() plt.imshow(test_Pred4[:, 20].reshape(28, -1), cmap="gray") plt.title("denoised_Sample 9 with_random_noise" + str(testnoise4) + "and train noise" + str(noise) + "is") plt.show() fig.savefig("Denoised_random noiseTest_Sample 9 op4") plt.plot(costs, label='Training cost') plt.title("training cost with learning rate =" + str(learning_rate) + "iterations" + str(num_iterations) + "noise :" + str(noise)) plt.show() fig.savefig("trainingCost") print("Total execution time: %s minutes!!" % ((time.time() - start_time) // 60))
def main(): # getting the subset dataset from MNIST # binary classification for digits 1 and 7 validation_costs = [] test_accuracies = [] # getting the subset dataset from MNIST data, data_label, test_data, test_label = \ mnist(noTrSamples=6000,noTsSamples=50,\ digit_range=[0,1,2,3,4,5,6,7,8,9],\ noTrPerClass=600, noTsPerClass=5) if par == 1: # initialize learning rate and num_iterations num_iterations = 3000 n_in, m = data.shape learning_rate = 0.2 net_dims = [784, 500, 784] costs_1, parameters_1, A_1 = two_layer_network(data, data_label,net_dims, \ num_iterations=num_iterations, learning_rate=learning_rate) learning_rate = 0.4 num_iterations = 1500 net_dims = [500, 200, 500] costs_2,parameters_2, A_2 = two_layer_network(A_1, data_label,net_dims, \ num_iterations=num_iterations, learning_rate=learning_rate) net_dims = [200, 100, 200] costs_3, parameters_3, A_3 = two_layer_network(A_2, data_label,net_dims, \ num_iterations=num_iterations, learning_rate=learning_rate) parameters = {} parameters["W1"] = parameters_1["W1"] parameters["b1"] = parameters_1["b1"] parameters["W2"] = parameters_2["W1"] parameters["b2"] = parameters_2["b1"] parameters["W3"] = parameters_3["W1"] parameters["b3"] = parameters_3["b1"] print(parameters["W1"].shape) print(parameters["W2"].shape) #print(parameters["W3"].shape) print(parameters) net_dims = [784, 500, 200, 100, 10] learning_rate = 1 num_iterations = 5000 costs_4, parameters_4 = multi_layer_network(parameters, data_last, data_last_label, net_dims, \ num_iterations=num_iterations, learning_rate=learning_rate) train_Pred = deepMultiClassNetwork_starter.classify(data, parameters_4) test_Pred = deepMultiClassNetwork_starter.classify( test_data, parameters_4) print("shape-DATA", data_label.shape) print("shape-TRAIN", train_Pred.shape) print("count:", np.count_nonzero(train_Pred - data_label)) trAcc = (1 - np.count_nonzero(train_Pred - data_label) / float(train_Pred.shape[1])) * 100 teAcc = (1 - np.count_nonzero(test_Pred - test_label) / float(test_Pred.shape[1])) * 100 print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc)) if par == 2: clf = svm.SVC(gamma='scale', decision_function_shape='ovo') clf.fit(data.T, data_label.T) out = clf.predict(test_data.T) print(out) print("Accuracy", (np.sum(out == test_label) / test_label.shape[1]) * 100) if par == 3: clf = LogisticRegression(random_state=0, solver='newton-cg', multi_class='multinomial').fit( data.T, data_label.T) score = clf.score(test_data.T, test_label.T) print(score) if par == 4: clf = GaussianNB() clf.fit(data.T, data_label.T) score = clf.score(test_data.T, test_label.T) print(score) clf = BernoulliNB() clf.fit(data.T, data_label.T) score = clf.score(test_data.T, test_label.T) print(score) clf = MultinomialNB() clf.fit(data.T, data_label.T) score = clf.score(test_data.T, test_label.T) print(score) if par == 5: clf = DecisionTreeClassifier(criterion='gini', random_state=0) clf.fit(data.T, data_label.T) score = clf.score(test_data.T, test_label.T) clf = RandomForestClassifier(n_estimators=100, max_depth=2, random_state=0) clf.fit(data.T, np.ravel(data_label.T)) score = clf.score(test_data.T, np.ravel(test_label.T)) print(score)
def main(): ''' Trains a multilayer network for MNIST digit classification (all 10 digits) To create a network with 1 hidden layer of dimensions 800 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800]" The network will have the dimensions [784,800,10] 784 is the input size of digit images (28pix x 28pix = 784) 10 is the number of digits To create a network with 2 hidden layers of dimensions 800 and 500 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800,500]" The network will have the dimensions [784,800,500,10] 784 is the input size of digit images (28pix x 28pix = 784) 10 is the number of digits ''' net_dims = ast.literal_eval(sys.argv[1]) net_dims.append(10) # Adding the digits layer with dimensionality = 10 print("Network dimensions are:" + str(net_dims)) # getting the subset dataset from MNIST train_data, train_label, test_data, test_label = \ mnist(noTrSamples=60000,noTsSamples=10000,\ digit_range=[0,1,2,3,4,5,6,7,8,9],\ noTrPerClass=6000, noTsPerClass=1000) train_data, validation_data, train_label, validation_label = train_test_split( train_data.T, train_label.T, test_size=10000, train_size=50000, random_state=42) learning_rate = 0.2 num_iterations = 1 epochs = 10 parameters = initialize_multilayer_weights(net_dims) process = psutil.Process(os.getpid()) total_time = 0 for j in range(0, epochs): start = time.time() for i in range(0, 10): print "EPOCH----------------------------->=", j print "BATCH----------------------------->=", i mini_train_data = train_data[i * 5000:(i * 5000) + 5000] mini_train_label = train_label[i * 5000:(i * 5000) + 5000] mini_train_data = mini_train_data.T mini_train_label = mini_train_label.T costs,Vcosts,parameters = multi_layer_network(i,parameters,mini_train_data,mini_train_label,validation_data.T,validation_label.T, net_dims, \ num_iterations=num_iterations, learning_rate=learning_rate) timetaken = time.time() - start total_time = total_time + timetaken print "TIME TAKEN=>", timetaken, " seconds" print(process.memory_info().rss) times.append(timetaken) print "TOTAL TIME TAKEN=", total_time # compute the accuracy for training set and testing set train_Pred = classify(train_data.T, parameters) test_Pred = classify(test_data, parameters) trAcc = calculateAccuracy(train_Pred.T, train_label.T) teAcc = calculateAccuracy(test_Pred.T, test_label) print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc)) print "MEMORY USAGE FOR TOTAL EPOCHS----->", (process.memory_info().rss * 0.001) / 784, "MB" ## CODE HERE to plot costs #train error vs iterations here x = [] for i in range(0, epochs): x.append(i) print costs print Vcosts print x plt.plot(x, costs) #,'ro') plt.plot(x, Vcosts) #,'b^') plt.show() plt.plot(x, times) plt.show()
def main(): net_dims = [784, 500, 100, 10] train_data, train_label, test_data, test_label, valid_data, valid_label = \ mnist(noTrSamples=5000, noTsSamples=1000, digit_range=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], noTrPerClass=500, noTsPerClass=100, noVdSamples=1000, noVdPerClass=100) num_iterations_list = [100, 500, 1000] # Add gradient method and its corresponding learning rate to the array. gradient_methods = [NO_MOMENTUM, MOMENTUM, NAG, RMSPROP, ADAM] learning_rates = [0.1, 0.05, 0.01, 0.001] for num_iterations in num_iterations_list: print(f"Running {num_iterations} Iterations") for learning_rate in learning_rates: all_costs = [] all_vcosts = [] time_per_algo = [] training_accuracy_per_algo = [] testing_accuracy_per_algo = [] for gradient_method in gradient_methods: start_time = datetime.datetime.now() mm = MultiLayerNeuralNetwork(net_dims, learning_rate, num_iterations, gradient_method) mm.fit(train_data, train_label, valid_data, valid_label) end_time = datetime.datetime.now() # compute the accuracy for training set and testing set Y_one_hot = one_hot(train_label, 10) t_one_hot = one_hot(test_label, 10) train_Pred = mm.predict(train_data, Y_one_hot) test_Pred = mm.predict(test_data, t_one_hot) trAcc = np.count_nonzero( train_Pred == train_label) / train_label.shape[1] * 100 teAcc = np.count_nonzero( test_Pred == test_label) / test_label.shape[1] * 100 print(f"Accuracy for training set is {trAcc:0.03f} %") print(f"Accuracy for testing set is {teAcc:0.03f} %") print( f"Time for the algo - {gradient_method} for {num_iterations} iterations was {(end_time - start_time).total_seconds()}" ) print("------------------------------------------------------") time_per_algo.append((end_time - start_time).total_seconds()) testing_accuracy_per_algo.append(teAcc) training_accuracy_per_algo.append(trAcc) all_vcosts.append(mm.validation_costs) all_costs.append(mm.costs) plot_costs_graph( all_costs, gradient_methods, f"outputs/brand/allcosts-{num_iterations}-{learning_rate}.png", learning_rate, False) plot_costs_graph( all_vcosts, gradient_methods, f"outputs/brand/allValidcosts-{num_iterations}-{learning_rate}.png", learning_rate, True) print(time_per_algo) print(testing_accuracy_per_algo) print(training_accuracy_per_algo) plot_bar_graph( time_per_algo, gradient_methods, f"outputs/brand/time-{num_iterations}-{learning_rate}.png", f"Plot of Time taken when using various algorithms for learning rate {learning_rate}", "Time") plot_bar_graph( testing_accuracy_per_algo, gradient_methods, f"outputs/brand/testing-accuracy-{num_iterations}-{learning_rate}.png", f"Plot of Testing Accuracy at LR: {learning_rate}", "Testing Accuracy") plot_bar_graph( training_accuracy_per_algo, gradient_methods, f"outputs/brand/training-accuracy-{num_iterations}-{learning_rate}.png", f"Plot of Training Accuracy at LR: {learning_rate}", "Training Accuracy")
def main(): n_in, m = 784, 784 n_h1 = 500 n_h2 = 100 net_dims = [n_in, n_h1, n_h2] #net_dims = ast.literal_eval( sys.argv[1] ) net_dims.append(10) # Adding the digits layer with dimensionality = 10 print("Network dimensions are:" + str(net_dims)) # getting the subset dataset from MNIST train_data, train_label, test_data, test_label = \ mnist(noTrSamples=6000,noTsSamples=1000,\ digit_range=[0,1,2,3,4,5,6,7,8,9],\ noTrPerClass=600, noTsPerClass=100) # initialize learning rate and num_iterations learning_rate = [0.00001, 0.0007, 0.0001] num_iterations = 300 decay_rate = 0.01 # batch_size = 10 batch_size = len(train_label) for x in learning_rate: import matplotlib.pyplot as plt costs, val_loss, parameters = multi_layer_network( train_data, train_label, net_dims, num_iterations=num_iterations, learning_rate=x, decay_rate=decay_rate, batch_size=batch_size) # compute the accuracy for training set and testing set train_Pred = classify(train_data, parameters) test_Pred = classify(test_data, parameters) test_error = ( (test_Pred[0] != test_label[0]).sum()) / test_Pred[0].sum() print("Test error: {0:0.4f} ".format(test_error)) trAcc = train_Pred - train_label teAcc = test_Pred - test_label trAcc[trAcc != 0] = 1 teAcc[teAcc != 0] = 1 trAcc = (1 - np.count_nonzero(train_Pred - train_label) / float(train_Pred.shape[1])) * 100 teAcc = (1 - np.count_nonzero(test_Pred - test_label) / float(test_Pred.shape[1])) * 100 print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc)) # train_loss, = plt.plot([x for x in range(num_iterations) if x % 10 == 0], costs) # val_loss, = plt.plot([x for x in range(num_iterations) if x % 10 == 0], val_loss) points = np.arange(0, 300) train_loss, = plt.plot(points, costs[:300]) val_loss, = plt.plot(val_loss) plt.xlabel("No Of Iterations") plt.ylabel("Cost/ val_loss") plt.title("Loss vs Number of Iterations with " + str(x) + " Learning rate") plt.legend((train_loss, val_loss), ("Train set Loss", "Validation Set Loss")) plt.show()
def main(): ''' Trains a multilayer network for MNIST digit classification (all 10 digits) To create a network with 1 hidden layer of dimensions 800 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800]" The network will have the dimensions [784,800,10] 784 is the input size of digit images (28pix x 28pix = 784) 10 is the number of digits To create a network with 2 hidden layers of dimensions 800 and 500 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800,500]" The network will have the dimensions [784,800,500,10] 784 is the input size of digit images (28pix x 28pix = 784) 10 is the number of digits ''' n_input, m = 784, 784 n_hidden1 = 500 n_hidden2 = 100 net_dims = [n_input, n_hidden1, n_hidden2] #net_dims = ast.literal_eval(sys.argv[1]) net_dims.append(10) # Adding the digits layer with dimensionality = 10 print("Network dimensions are:" + str(net_dims)) # getting the subset dataset from Fashion_mnist orig_train_data, orig_train_label, test_data, test_label = \ mnist(noTrSamples=6000, noTsSamples=1000, digit_range=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], noTrPerClass=600, noTsPerClass=100) train_data = orig_train_data[:, :500] train_label = orig_train_label[:, :500] validation_data = orig_train_data[:, 500:600] validation_label = orig_train_label[:, 500:600] for i in range(600, 6000, 600): train_data = np.hstack((train_data, orig_train_data[:, i:i + 500])) train_label = np.hstack((train_label, orig_train_label[:, i:i + 500])) validation_data = np.hstack( (validation_data, orig_train_data[:, i + 500:i + 600])) validation_label = np.hstack( (validation_label, orig_train_label[:, i + 500:i + 600])) learning_rate = 0.01 decay_rate = 0.01 num_iterations = 500 batch_size = 10 costs, parameters, vcosts = multi_layer_network( train_data, train_label, validation_data, validation_label, net_dims, num_iterations=num_iterations, learning_rate=learning_rate, decay_rate=decay_rate, batch_size=batch_size) # compute the accuracy for training set and testing set train_Pred = classify(train_data, parameters) test_Pred = classify(test_data, parameters) val_Pred = classify(validation_data, parameters) trAcc = (1 - np.count_nonzero(train_Pred - train_label) / float(train_Pred.shape[1])) * 100 teAcc = (1 - np.count_nonzero(test_Pred - test_label) / float(test_Pred.shape[1])) * 100 valAcc = (1 - np.count_nonzero(val_Pred - validation_label) / float(val_Pred.shape[1])) * 100 print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc)) print("Accuracy for validation set is {0:0.3f} %".format(valAcc)) X = np.arange(0, 500) plt.plot(X, costs, label="Train Cost") plt.plot(X, vcosts, label="Validation Cost") plt.xlabel("No Of Iterations") plt.ylabel("Training Cost / Validation Cost") plt.title("Cost vs Number of Iterations with " + str(learning_rate) + " Learning rate") plt.legend() plt.show()
def sgd(cost, params, lr=0.05): grads = T.grad(cost=cost, wrt=params) updates = [] for p, g in zip(params, grads): updates.append([p, p - g * lr]) return updates def model(X, w_h, w_o): h = T.nnet.sigmoid(T.dot(X, w_h)) pyx = T.nnet.softmax(T.dot(h, w_o)) return pyx trX, teX, trY, teY = mnist(onehot=True) X = T.fmatrix() Y = T.fmatrix() w_h = init_weights((784, 625)) w_o = init_weights((625, 10)) py_x = model(X, w_h, w_o) y_x = T.argmax(py_x, axis=1) cost = T.mean(T.nnet.categorical_crossentropy(py_x, Y)) params = [w_h, w_o] updates = sgd(cost, params) train = theano.function(inputs=[X, Y],
def main(): ''' Trains a multilayer network for MNIST digit classification (all 10 digits) To create a network with 1 hidden layer of dimensions 800 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800]" The network will have the dimensions [784,800,10] 784 is the input size of digit images (28pix x 28pix = 784) 10 is the number of digits To create a network with 2 hidden layers of dimensions 800 and 500 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800,500]" The network will have the dimensions [784,800,500,10] 784 is the input size of digit images (28pix x 28pix = 784) 10 is the number of digits ''' net_dims = ast.literal_eval(sys.argv[1]) net_dims.append(10) # Adding the digits layer with dimensionality = 10 print("Network dimensions are:" + str(net_dims)) # getting the subset dataset from MNIST train_data, train_label, test_data, test_label = \ mnist(noTrSamples=6000,noTsSamples=1000,\ digit_range=[0,1,2,3,4,5,6,7,8,9],\ noTrPerClass=600, noTsPerClass=100) valid_data, valid_label = train_data[:, 5000:6000], train_label[:, 5000:6000] # initialize learning rate and num_iterations learning_rate = 10.0 num_iterations = 1000 num_train_samples = 5000 num_validate_samples = 1000 num_test_samples = 1000 costs, parameters ,costs_valid = multi_layer_network(train_data, train_label,valid_data,valid_label, net_dims, \ num_iterations=num_iterations, learning_rate= learning_rate, decay_rate = 0.0) # compute the accuracy for training set and testing set train_Pred = classify(train_data, parameters) test_Pred = classify(test_data, parameters) #print (train_label.shape) #print(train_Pred.shape) #print (train_label) #print(train_Pred) train_label_new = one_hot(train_label.astype(int), 10) test_label_new = one_hot(test_label.astype(int), 10) result_train = np.sum(abs(train_Pred - train_label_new.T)) print("Train error is : %f" % (result_train)) result_test = np.sum(abs(test_Pred - test_label_new.T)) print("Test error is : %f" % (result_test)) trAcc = 1 / num_train_samples * np.sum(num_train_samples - result_train) * 100 teAcc = 1 / num_test_samples * np.sum(num_test_samples - result_test) * 100 print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc)) ### CODE HERE to plot costs x = range(0, int(num_iterations / 10)) plt.plot(x, costs) plt.plot(x, costs_valid) plt.xlabel('iterations') plt.ylabel('Costs') plt.title('Training and validation') plt.show()
def main(): # getting the subset dataset from MNIST # binary classification for digits 1 and 7 hiddenNodes = [100, 200, 500] digit_range = [1, 7] data, label, test_data, test_label = \ mnist(noTrSamples=2400,noTsSamples=1000,\ digit_range=digit_range,\ noTrPerClass=1200, noTsPerClass=500) train_data = data[:, 0:1000] train_data2 = data[:, 1400:2400] train_data = np.column_stack((train_data, train_data2)) train_label = label[:, 0:1000] train_label2 = label[:, 1400:2400] train_label = np.column_stack((train_label, train_label2)) validation_data = data[:, 1000:1400] validation_label = label[:, 1000:1400] # intialially its 1 and 7 #convert to binary labels # 0 -> 1 # 1 -> 7 train_label[train_label == digit_range[0]] = 0 train_label[train_label == digit_range[1]] = 1 test_label[test_label == digit_range[0]] = 0 test_label[test_label == digit_range[1]] = 1 validation_label[validation_label == digit_range[0]] = 0 validation_label[validation_label == digit_range[1]] = 1 mxValidationAccuracy = 0 testError = 0 nodes = 200 mxvcost = 99 fig, axs = plt.subplots(1, 3, constrained_layout=True) fig.suptitle('\nTraining cost v iterations\n') i = 0 for n_h in hiddenNodes: if (n_h == 400): print "\nVarying hidden nodes to check accuracy" print "Number of hidden nodes :", n_h n_in, m = train_data.shape # n_in == 784 , m==2000 n_fin = 1 # for 500 nodes # NN -> 784->500_->1 net_dims = [n_in, n_h, n_fin] # initialize learning rate and num_iterations learning_rate = 0.1 num_iterations = 1000 # print train_label costs,vcosts, parameters,A2 = two_layer_network(train_data, train_label, net_dims, \ validation_data,validation_label,num_iterations=num_iterations, learning_rate=learning_rate) # compute the accuracy for training set and testing set train_Pred = classify(train_data, parameters) train_Accuracy = calculateAccuracy(train_label, train_Pred) validation_Pred = classify(validation_data, parameters) validation_Accuracy = calculateAccuracy(validation_label, validation_Pred) test_Pred = classify(test_data, parameters) test_Accuracy = calculateAccuracy(test_label, test_Pred) print "Accuracy for training set is ", train_Accuracy print "Accuracy for validation set is ", validation_Accuracy print "Accuracy for testing set is ", test_Accuracy print "Test Error is ", 100 - test_Accuracy, "%", "\n" if (float(vcosts[100]) < float(mxvcost)): mxvcost = vcosts[100] testError = test_Accuracy nodes = n_h mxValidationAccuracy = validation_Accuracy # CODE HERE TO PLOT costs vs iterations iterations = [k for k in range(0, 1001, 10)] axs[i].plot(iterations, costs, 'r', label="Training Cost") axs[i].plot(iterations, vcosts, 'b', label="Validation Cost") axs[i].set_title("Hidden nodes :" + str(n_h)) axs[i].set_xlabel('Iterations') axs[i].set_ylabel('Costs') axs[i].legend() i += 1 # axs[1].plot(iterations,vcosts) # axs[1].set_xlabel('Iterations') # axs[1].set_title('\nValidation cost vs iterations\n"') # axs[1].set_ylabel('Costs') plt.show() print "Best Validation Cost is: ", mxvcost, " where number of hidden nodes :", nodes print "Validation accuracy for this architecture :", mxValidationAccuracy print "Test Accuracy for this architecture :", testError print "\n"
#coding=utf-8 import tensorflow as tf from load_mnist import mnist mnist = mnist() mnist.load_mnist('../../data', 'train') iter_num = 200000 #迭代次数 learning_rate = 0.001 #学习率 batch_size = 64 #每次训练采用的样本数 display = 20 #输出的轮数 input_num = 28 * 28 #输入图像的像素大小 class_num = 10 #类别的个数 drop_out = 0.8 #剪枝的比率 X = tf.placeholder(tf.float32, shape=(None, input_num)) y = tf.placeholder(tf.float32, shape=[None, class_num]) keep_prob = tf.placeholder(tf.float32) def conv(input, w, b, name): return tf.nn.relu(tf.nn.bias_add( tf.nn.conv2d(input, w, strides=[1, 1, 1, 1], padding='SAME'), b), name=name) def maxpooling(input, k, name): return tf.nn.max_pool(input, ksize=[1, k, k, 1], strides=[1, k, k, 1],
def main(): ''' Trains a multilayer network for MNIST digit classification (all 10 digits) To create a network with 1 hidden layer of dimensions 800 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800]" The network will have the dimensions [784,800,10] 784 is the input size of digit images (28pix x 28pix = 784) 10 is the number of digits To create a network with 2 hidden layers of dimensions 800 and 500 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800,500]" The network will have the dimensions [784,800,500,10] 784 is the input size of digit images (28pix x 28pix = 784) 10 is the number of digits ''' net_dims = ast.literal_eval(sys.argv[1]) net_dims.append(10) # Adding the digits layer with dimensionality = 10 print("Network dimensions are:" + str(net_dims)) # getting the subset dataset from MNIST Original_train_data, Original_train_label, test_data, test_label = \ mnist(noTrSamples=6000, noTsSamples=1000, \ digit_range=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], \ noTrPerClass=600, noTsPerClass=100) train_data = Original_train_data[:, :500] train_label = Original_train_label[:, :500] validation_data = Original_train_data[:, 500:600] validation_label = Original_train_label[:, 500:600] for i in range(600, 6000, 600): train_data = np.hstack((train_data, Original_train_data[:, i:i + 500])) train_label = np.hstack((train_label, Original_train_label[:, i:i + 500])) validation_data = np.hstack((validation_data, Original_train_data[:, i + 500:i + 600])) validation_label = np.hstack((validation_label, Original_train_label[:, i + 500:i + 600])) learning_rate = 0.5 num_iterations = 500 costs, parameters,valid_costs = multi_layer_network(train_data, train_label,validation_data,validation_label, net_dims, \ num_iterations=num_iterations, learning_rate=learning_rate) # compute the accuracy for training set and testing set train_Pred = classify(train_data, parameters) test_Pred = classify(test_data, parameters) val_Pred = classify(validation_data, parameters) trAcc = (1 - np.count_nonzero(train_Pred - train_label) / float(train_Pred.shape[1])) * 100 teAcc = (1 - np.count_nonzero(test_Pred - test_label) / float(test_Pred.shape[1])) * 100 valAcc = (1 - np.count_nonzero(val_Pred - validation_label) / float(val_Pred.shape[1])) * 100 print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc)) print("Accuracy for validation set is {0:0.3f} %".format(valAcc)) X = range(0, 500, 10) plt.plot(X, costs, label="train data") plt.plot(X,valid_costs,label="validation") plt.xlabel("Iterations") plt.ylabel("Cost") plt.legend() plt.show()
def main(): start_time = time.time() train_data, train_label, test_data, test_label = mnist( noTrSamples=60000, noTsSamples=10000, digit_range=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], noTrPerClass=6000, noTsPerClass=1000) # print(param) n_rows = train_data.shape[0] n_cols = train_data.shape[1] test_rows = test_data.shape[0] test_cols = test_label.shape[1] mean = 0.0 stddev = 0.2 noise = np.random.normal(mean, stddev, (n_rows, n_cols)) trX_noisy = train_data + noise noise = np.random.normal(mean, stddev, (test_rows, test_cols)) tsX_noisy = test_data + noise # fig = plt.figure() # plt.imshow(train_data[:, 999].reshape(28, -1)) # plt.title("Training_Sample") # plt.show() # fig.savefig("Training_Sample") # # fig = plt.figure() # plt.imshow(trX_noisy[:, 999].reshape(28, -1)) # plt.title("Noisy_Training_Sample") # plt.show() # fig.savefig("Noisy_Training_Sample") n_in, m = train_data.shape n_fin, m = train_data.shape n_h = 1000 net_dims = [n_in, n_h, n_fin] # initialize learning rate and num_iterations learning_rate = 1 num_iterations = 1000 costs, parameters = two_layer_network(trX_noisy, train_data, net_dims, num_iterations=num_iterations, learning_rate=learning_rate) # compute the accuracy for training set and testing set # train_Pred = classify(train_data, parameters) fig = plt.figure() plt.imshow(test_data[:, 9999].reshape(28, -1), cmap=plt.cm.binary) plt.title("Test_Sample") plt.show() fig.savefig("Test_Sample") fig = plt.figure() plt.imshow(tsX_noisy[:, 9999].reshape(28, -1), cmap=plt.cm.binary) plt.title("Noisy_Test_Sample") plt.show() fig.savefig("Noisy_Test_Sample") fig = plt.figure() test_Pred = denoise(tsX_noisy, parameters) plt.imshow(test_Pred[:, 9999].reshape(28, -1), cmap=plt.cm.binary) plt.title("Denoised_Test_Sample") plt.show() fig.savefig("Denoised_Test_Sample") print("Total execution time: %s minutes!!" % ((time.time() - start_time) // 60))
import matplotlib.pyplot as plt from sklearn import datasets, svm, metrics from load_mnist import mnist import numpy as np digit_range = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] train_data, train_label, val_data, val_label, test_data, test_label = \ mnist(noTrSamples=50, noValSamples=0, noTsSamples=1000,\ digit_range=digit_range,\ noTrPerClass=5, noValPerClass=0, noTsPerClass=100) train_data = np.transpose(train_data) train_label = np.transpose(train_label) train_label = np.squeeze(train_label, axis=1) test_data = np.transpose(test_data) test_label = np.transpose(test_label) test_label = np.squeeze(test_label, axis=1) classifier = svm.SVC(C=5, gamma=0.05) classifier.fit(train_data, train_label) predicted = classifier.predict(test_data) accuracy = metrics.accuracy_score(test_label, predicted)
def main(): ''' Trains a multilayer network for MNIST digit classification (all 10 digits) To create a network with 1 hidden layer of dimensions 800 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800]" The network will have the dimensions [784,800,10] 784 is the input size of digit images (28pix x 28pix = 784) 10 is the number of digits To create a network with 2 hidden layers of dimensions 800 and 500 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800,500]" The network will have the dimensions [784,800,500,10] 784 is the input size of digit images (28pix x 28pix = 784) 10 is the number of digits ''' net_dims = ast.literal_eval( sys.argv[1] ) net_dims.append(10) # Adding the digits layer with dimensionality = 10 print("Network dimensions are:" + str(net_dims)) # getting the subset dataset from MNIST train_data, train_label, test_data, test_label = \ mnist(noTrSamples=60000,noTsSamples=10000,\ digit_range=[0,1,2,3,4,5,6,7,8,9],\ noTrPerClass=6000, noTsPerClass=1000) train_data, validation_data,train_label, validation_label= train_test_split(train_data.T,train_label.T,test_size=10000,train_size=50000,random_state=42) learning_rate = 0.2 num_iterations = 1 epochs=1000 parameters = initialize_multilayer_weights(net_dims) costvalues, costvalues_valid, times = [], [], [] for j in range(0,epochs): start = time.process_time() for i in range(0,10): print ("EPOCH------------->=",j) print ("BATCH------------->=",i) mini_train_data=train_data[i*5000 : (i*5000)+5000] mini_train_label=train_label[i*5000 : (i*5000)+5000] mini_train_data=mini_train_data.T mini_train_label=mini_train_label.T costs,Vcosts,parameters = multi_layer_network(parameters,mini_train_data,mini_train_label,validation_data.T,validation_label.T, net_dims, num_iterations=num_iterations, learning_rate=learning_rate) stop = time.process_time() timetaken = stop - start times.append(timetaken) costvalues.append(costs) costvalues_valid.append(Vcosts) print("time taken for this epoch --->= ", timetaken) train_Pred = classify(train_data.T, parameters) test_Pred = classify(test_data, parameters) trAcc = calculateAccuracy(train_Pred.T,train_label.T) teAcc = calculateAccuracy(test_Pred.T,test_label) print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc)) print("Memory Usage : ",int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)/(784*1024), "MB") plt.plot(np.squeeze(costvalues)); plt.plot(np.squeeze(costvalues_valid)); plt.show() plt.plot(np.squeeze(times)); plt.show()
def main(): ''' Trains a multilayer network for MNIST digit classification (all 10 digits) To create a network with 1 hidden layer of dimensions 800 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800]" The network will have the dimensions [784,800,10] 784 is the input size of digit images (28pix x 28pix = 784) 10 is the number of digits To create a network with 2 hidden layers of dimensions 800 and 500 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800,500]" The network will have the dimensions [784,800,500,10] 784 is the input size of digit images (28pix x 28pix = 784) 10 is the number of digits ''' net_dims = ast.literal_eval(sys.argv[1]) net_dims.append(10) # Adding the digits layer with dimensionality = 10 print("Network dimensions are:" + str(net_dims)) # getting the subset dataset from MNIST data, label, test_data, test_label = \ mnist(noTrSamples=6000, noTsSamples=1000, digit_range=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], noTrPerClass=600, noTsPerClass=100) for i in range(10): if i == 0: train_data = data[:, :500] val_data = data[:, 500:600] train_label = label[:, :500] val_label = label[:, 500:600] else: train_data = np.concatenate( [train_data, data[:, 600 * (i):(500 + (i) * 600)]], axis=1) val_data = np.concatenate( [val_data, data[:, (500 + (i) * 600):(500 + (i) * 600) + 100]], axis=1) train_label = np.concatenate( [train_label, label[:, 600 * (i):(500 + (i) * 600)]], axis=1) val_label = np.concatenate([ val_label, label[:, (500 + (i) * 600):(500 + (i) * 600) + 100] ], axis=1) # initialize learning rate and num_iterations learning_rate = 1.5 num_iterations = 500 cost_tr, cost_val, parameters = multi_layer_network( train_data, train_label, val_data, val_label, net_dims, num_iterations=num_iterations, learning_rate=learning_rate) # compute the accuracy for training set and testing set train_Pred = classify(train_data, parameters) test_Pred = classify(test_data, parameters) trAcc = accuracy(train_Pred, train_label) teAcc = accuracy(test_Pred, test_label) print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc)) plt.xlabel("iterations") plt.ylabel("cost") plt.title("Multiclass - iterations vs cost for train and validation data") plt.plot(cost_tr, "-g", label='train') plt.plot(cost_val, ":b", label='val') plt.legend() plt.show()
# python 3 import matplotlib.pyplot as plt import sys from load_mnist import mnist from sklearn import svm import warnings # Save all the Print Statements in a Log file. print_out = sys.stdout log_file = open("BaseLine_SVM_Results.txt", "a") sys.stdout = log_file train_data, train_label, test_data, test_label = mnist( noTrSamples=60000, noTsSamples=10000, digit_range=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], noTrPerClass=6000, noTsPerClass=1000) # print(train_data.shape, test_data.shape) # plt.figure() # plt.imshow(train_data[:, 0].reshape(28, -1), cmap=plt.cm.binary) # plt.colorbar() # plt.grid(False) # plt.show() print('\nSVM Classifier with gamma = 0.1; Kernel = polynomial') with warnings.catch_warnings(): warnings.simplefilter("ignore") clf = svm.SVC(gamma=0.1, kernel='poly') clf.fit(train_data.T, train_label.T)
def main(): ''' Trains a multilayer network for MNIST digit classification (all 10 digits) To create a network with 1 hidden layer of dimensions 800 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800]" The network will have the dimensions [784,800,10] 784 is the input size of digit images (28pix x 2 8pix = 784) 10 is the number of digits To create a network with 2 hidden layers of dimensions 800 and 500 Run the progam as: python deepMultiClassNetwork_starter.py "[784,800,500]" The network will have the dimensions [784,800,500,10] 784 is the input size of digit images (28pix x 28pix = 784) 10 is the number of digits ''' # getting the subset dataset from MNIST data, label, test_data, test_label = \ mnist(noTrSamples=6000,noTsSamples=1000,\ digit_range=[0,1,2,3,4,5,6,7,8,9],\ noTrPerClass=600, noTsPerClass=100) # parse data into training and validation sets # training data train_data = np.array([]) train_label = np.array([]) train_data = data[:, 0:500] train_label1 = label[:, 0:500] for i in range(600, 6000, 600): # print i temp = data[:, i:i + 500] temp2 = label[:, i:i + 500] # print temp.shape train_data = np.column_stack((train_data, temp)) train_label1 = np.column_stack((train_label1, temp2)) # train data -> 784*5000 # reshape train_lable from 1*5000 to 10*5000 # since instead of train_label[sample]=digit it should be train_label[sample][digit] = 1/0 column = train_label1.shape[1] mainList = [] for i in range(0, column): l = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] temp = int(train_label1[0][i]) l[temp] = 1 mainList.append(l) train_label = np.array(mainList) train_label = train_label.T # print "Train labels",train_label.shape # validation data validation_data = np.array([]) validation_label = np.array([]) validation_data = data[:, 500:600] validation_label1 = label[:, 500:600] for i in range(1100, 6000, 600): # print i temp = data[:, i:i + 100] temp2 = label[:, i:i + 100] # print temp.shape validation_data = np.column_stack((validation_data, temp)) validation_label1 = np.column_stack((validation_label1, temp2)) # train data -> 784*5000 # reshape train_lable from 1*5000 to 10*5000 # since instead of train_label[sample]=digit it should be train_label[sample][digit] = 1/0 column = validation_label1.shape[1] mainList = [] for i in range(0, column): l = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] temp = int(validation_label1[0][i]) l[temp] = 1 mainList.append(l) validation_label = np.array(mainList) validation_label = validation_label.T # validation data -> 784*1000 # confirm data # a =[0,0,0,0,0,0,0,0,0,0] # for i in validation_label[0]: # a[int(i)]+=1 # print a # initialize learning rate and num_iterations # varying learning rates. num_iterations = 1000 rates = [0.1, 0.2, 0.5, 1.0, 10] net_dims = ast.literal_eval(sys.argv[1]) # print net_dims net_dims.append(10) # Adding the digits layer with dimensionality = 10 print "Network dimensions are:" + str(net_dims) ax1 = plt.subplot2grid(shape=(2, 6), loc=(0, 0), colspan=2) ax2 = plt.subplot2grid((2, 6), (0, 2), colspan=2) ax3 = plt.subplot2grid((2, 6), (0, 4), colspan=2) ax4 = plt.subplot2grid((2, 6), (1, 1), colspan=2) ax5 = plt.subplot2grid((2, 6), (1, 3), colspan=2) l = [ax1, ax2, ax3, ax4, ax5] j = 0 for learning_rate in rates: print "Learning Rate: ", learning_rate costs, vcosts,parameters = multi_layer_network(train_data, train_label,validation_data,validation_label, net_dims, \ num_iterations=num_iterations, learning_rate=learning_rate) # compute the accuracy for training set and testing set train_Pred = classify(train_data, parameters) test_Pred = classify(test_data, parameters) validation_Pred = classify(validation_data, parameters) trAcc = calculateAccuracy(train_label1, train_Pred) teAcc = calculateAccuracy(test_label, test_Pred) vaAcc = calculateAccuracy(validation_label1, validation_Pred) print "Accuracy for training set is {0:0.3f} %".format(trAcc) print "Accuracy for Validation set is {0:0.3f} %".format(vaAcc) print "Accuracy for testing set is {0:0.3f} %".format(teAcc) ### CODE HERE to plot costs iterations = [i for i in range(0, 1001, 10)] # ,iterations,vcosts,'b',label='Validation cost' l[j].plot(iterations, costs, 'r', label='Training cost') l[j].plot(iterations, vcosts, 'b', label='Validation cost') title = "Learning rate: ", learning_rate l[j].set_title(title) l[j].set_xlabel('Iterations') l[j].set_ylabel('Costs') l[j].legend() j += 1 plt.show()
def main(): start_time = time.time() train_data, train_label, test_data, test_label = mnist() # print(param) n_rows = train_data.shape[0] n_cols = train_data.shape[1] test_rows = test_data.shape[0] test_cols = test_label.shape[1] mean = 0.0 stddev = 0.4 noise = np.random.normal(mean, stddev, (n_rows, n_cols)) trX_noisy = train_data + noise noise = np.random.normal(mean, stddev, (test_rows, test_cols)) tsX_noisy = test_data + noise # fig = plt.figure() # plt.imshow(train_data[:, 999].reshape(28, -1)) # plt.title("Training_Sample") # plt.show() # fig.savefig("Training_Sample") # # fig = plt.figure() # plt.imshow(trX_noisy[:, 999].reshape(28, -1)) # plt.title("Noisy_Training_Sample") # plt.show() # fig.savefig("Noisy_Training_Sample") n_in, m = train_data.shape n_fin, m = train_data.shape n_h = 1000 net_dims = [n_in, n_h, n_fin] # initialize learning rate and num_iterations learning_rate = 1 num_iterations = 1000 costs, parameters = two_layer_network(trX_noisy, train_data, net_dims, num_iterations=num_iterations, learning_rate=learning_rate) # compute the accuracy for training set and testing set # train_Pred = classify(train_data, parameters) fig = plt.figure() plt.imshow(test_data[:, 99].reshape(28, -1)) plt.title("Test_Sample") plt.show() fig.savefig("Test_Sample") fig = plt.figure() plt.imshow(tsX_noisy[:, 99].reshape(28, -1)) plt.title("Noisy_Test_Sample") plt.show() fig.savefig("Noisy_Test_Sample") fig = plt.figure() test_Pred = denoise(tsX_noisy, parameters) print("error of test sample" ,(error(test_Pred,test_data))) plt.imshow(test_Pred[:, 99].reshape(28, -1)) plt.title("Denoised_Test_Sample") plt.show() fig.savefig("Denoised_Test_Sample") plt.plot(costs, label = 'Training cost')
l2a = rectify(conv2d(l1, w2)) l2 = max_pool_2d(l2a, (2, 2)) l2 = dropout(l2, p_drop_conv) l3a = rectify(conv2d(l2, w3)) l3b = max_pool_2d(l3a, (2, 2)) l3 = T.flatten(l3b, outdim=2) l3 = dropout(l3, p_drop_conv) l4 = rectify(T.dot(l3, w4)) l4 = dropout(l4, p_drop_hidden) pyx = softmax(T.dot(l4, w_o)) return l1, l2, l3, l4, pyx trX, teX, trY, teY = mnist(onehot=True) trX = trX.reshape(-1, 1, 28, 28) teX = teX.reshape(-1, 1, 28, 28) X = T.tensor4(dtype='float64') Y = T.matrix() w = init_weights((32, 1, 3, 3)) w2 = init_weights((64, 32, 3, 3)) w3 = init_weights((128, 64, 3, 3)) w4 = init_weights((128 * 3 * 3, 625)) w_o = init_weights((625, 10)) noise_l1, noise_l2, noise_l3, noise_l4, noise_py_x = model(X, w, w2, w3, w4, 0.2, 0.5) l1, l2, l3, l4, py_x = model(X, w, w2, w3, w4, 0., 0.)
def main(): # getting the subset dataset from MNIST # binary classification for digits 1 and 7 digit_range = [1, 7] data, data_label, test_data, test_label = \ mnist(noTrSamples=2400,noTsSamples=1000,\ digit_range=digit_range,\ noTrPerClass=1200, noTsPerClass=500) print(data_label) validation_costs = [] test_accuracies = [] train_data = np.concatenate((data[:, :1000], data[:, 1200:2200]), axis=1) val_data = np.concatenate((data[:, 1000:1200], data[:, 2200:2400]), axis=1) train_label = np.concatenate( (data_label[0][:1000], data_label[0][1200:2200])) val_label = np.concatenate( (data_label[0][1000:1200], data_label[0][2200:2400])) #convert to binary labels train_label[train_label == digit_range[0]] = 0 train_label[train_label == digit_range[1]] = 1 test_label[test_label == digit_range[0]] = 0 test_label[test_label == digit_range[1]] = 1 val_label[val_label == digit_range[0]] = 0 val_label[val_label == digit_range[1]] = 1 n_in, m = train_data.shape n_fin = 1 n_h = [100, 200, 500] count = 1 num_iterations = 1000 iteration = [k for k in range(0, num_iterations, 10)] #fig = plt.figure(figsize=(15, 10)) fig = plt.figure() for n, j in enumerate(n_h): net_dims = [n_in, j, n_fin] # initialize learning rate and num_iterations learning_rate = 0.01 out = [] out_t = [] out_v = [] print("Number of neurons in the hidden layer:", j) print() print("Training set") costs, parameters = two_layer_network(train_data, train_label, net_dims, \ num_iterations=num_iterations, learning_rate=learning_rate) print( "------------------------------------------------------------------------------" ) print("Validation set") costs_val,parameters_val = two_layer_network(val_data, val_label, net_dims, \ num_iterations=num_iterations, learning_rate=learning_rate) validation_costs.append(costs_val[-1]) print( "------------------------------------------------------------------------------" ) # compute the accuracy for training set and testing set train_Pred = classify(train_data, parameters) val_Pred = classify(val_data, parameters) test_Pred = classify(test_data, parameters) trAcc = None teAcc = None output_test = (test_label == test_Pred) teAcc = (np.sum(output_test == True) / test_label.shape[1]) * 100 test_accuracies.append(teAcc) output = (train_label == train_Pred) trAcc = (np.sum(output == True) / train_label.shape[0]) * 100 output_val = (val_label == val_Pred) valAcc = (np.sum(output_val == True) / val_label.shape[0]) * 100 print("Accuracy:") print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for validation set is {0:0.3f} %".format(valAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc)) print( "------------------------------------------------------------------------------" ) # CODE HERE TO PLOT costs vs iterations if j == 100: sub1 = fig.add_subplot(1, 3, 1) sub1.plot(iteration, costs, label="Train") sub1.plot(iteration, costs_val, label="Validation") sub1.legend() sub1.set_title('100 Neurons in the hidden layer') sub1.set_xlabel('Iterations') sub1.set_ylabel('Cost') if j == 200: sub2 = fig.add_subplot(1, 3, 2) sub2.plot(iteration, costs, label="Train") sub2.plot(iteration, costs_val, label="Validation") sub2.legend() sub2.set_title('200 Neurons in the hidden layer') sub2.set_xlabel('Iterations') sub2.set_ylabel('Cost') if j == 500: sub3 = fig.add_subplot(1, 3, 3) sub3.plot(iteration, costs, label="Train") sub3.plot(iteration, costs_val, label="Validation") sub3.legend() sub3.set_title('500 Neurons in the hidden layer') sub3.set_xlabel('Iterations') sub3.set_ylabel('Cost') plt.show() idx = validation_costs.index(min(validation_costs)) print("Mimimum validation cost:", min(validation_costs), "observed with ", n_h[idx], "hidden layers") print( "Test accuracy using the architecture with the best validation cost:", test_accuracies[idx])
def main(): # getting the subset dataset from MNIST net_dims = ast.literal_eval(sys.argv[1]) net_dims.append(1) digit_range = [1, 7] train_data, train_label, data, label = \ mnist(noTrSamples=2000, noTsSamples=1400, \ digit_range=digit_range, \ noTrPerClass=1000, noTsPerClass=700) test_data = np.concatenate((data[:, :500], data[:, 700:1200]), axis=1) val_data = np.concatenate((data[:, 500:700], data[:, 1200:1400]), axis=1) test_label = np.concatenate((label[:, :500], label[:, 700:1200]), axis=1) val_label = np.concatenate((label[:, 500:700], label[:, 1200:1400]), axis=1) val_label[val_label == digit_range[0]] = 0 val_label[val_label == digit_range[1]] = 1 train_label[train_label == digit_range[0]] = 0 train_label[train_label == digit_range[1]] = 1 test_label[test_label == digit_range[0]] = 0 test_label[test_label == digit_range[1]] = 1 n_in, m = train_data.shape n_fin = 1 n_h = 500 net_dims = [n_in, n_h, n_fin] # initialize learning rate and num_iterations learning_rate = 0.1 num_iterations = 1000 costs, parameters,costs1 = two_layer_network(train_data, train_label,val_data, val_label, net_dims, \ num_iterations=num_iterations, learning_rate=learning_rate) # compute the accuracy for training set and testing set train_Pred = classify(train_data, parameters) test_Pred = classify(test_data, parameters) val_Pred=classify(val_data,parameters) m1 = train_data.shape[1] m2 = test_data.shape[1] m3 = val_data.shape[1] trAcc = 100 - (np.sum(np.absolute(train_label - train_Pred)) / m1) * 100 teAcc = 100 - (np.sum(np.absolute(test_label - test_Pred)) / m2) * 100 valAcc = 100 - (np.sum(np.absolute(val_label - val_Pred)) / m3) * 100 print("Accuracy for training set is {0:0.3f} %".format(trAcc)) print("Accuracy for testing set is {0:0.3f} %".format(teAcc)) print("Accuracy for validation set is {0:0.3f} %".format(valAcc)) points = np.arange(0, 100) plt.plot(points, costs, label='train') plt.plot(points, costs1, label='validation') plt.xlabel('iterations') plt.ylabel('cost') plt.title("Error vs iterations") plt.legend() plt.show() plt.savefig("Error vs iterations")