def main():
# getting the subset dataset from MNIST
# binary classification for digits 1 and 7
digit_range = [1, 7]
data, label, test_data, test_label = \
mnist(noTrSamples=2400,noTsSamples=1000,\
digit_range=digit_range,\
noTrPerClass=1200, noTsPerClass=500)
temp1, temp2, val_data, val_label = mnist(noTrSamples=2,
noTsSamples=400,
digit_range=digit_range,
noTrPerClass=1,
noTsPerClass=200)
train_data = np.concatenate([data[:, :1000], data[:, 1200:2200]], axis=1)
val_data = np.concatenate([data[:, 1000:1200], data[:, 2200:2400]], axis=1)
train_label = np.concatenate([label[:, :1000], label[:, 1200:2200]],
axis=1)
val_label = np.concatenate([label[:, 1000:1200], label[:, 2200:2400]],
axis=1)
#convert to binary labels
train_label[train_label == digit_range[0]] = 0
train_label[train_label == digit_range[1]] = 1
test_label[test_label == digit_range[0]] = 0
test_label[test_label == digit_range[1]] = 1
val_label[val_label == digit_range[0]] = 0
val_label[val_label == digit_range[1]] = 1
n_in, m = train_data.shape
n_fin = 1
n_h = 500
net_dims = [n_in, n_h, n_fin]
# initialize learning rate and num_iterations
learning_rate = 0.01
num_iterations = 500
costs_tr, costs_val, parameters_tr = two_layer_network(train_data, train_label, val_data, val_label, net_dims, \
num_iterations=num_iterations, learning_rate=learning_rate)
# compute the accuracy for training set and testing set
train_Pred = classify(train_data, parameters_tr)
test_Pred = classify(test_data, parameters_tr)
trAcc = accuracy(train_Pred, train_label)
teAcc = accuracy(test_Pred, test_label)
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
# CODE HERE TO PLOT costs vs iterations
plt.xlabel("iterations")
plt.ylabel("cost")
plt.title("BINARY - iterations vs cost for train and validation data")
plt.plot(costs_tr, "-g", label='train')
plt.plot(costs_val, ":b", label='val')
plt.legend()
plt.show()
def main():
# load the data from external load_mnist.py script
train_X, train_y, test_X, test_y = mnist(noTrSamples=400,
noTsSamples=100,
digit_range=[5, 8],
noTrPerClass=200,
noTsPerClass=50)
# get pca data
pca_train_X, e_pca_train_X = pca(train_X, dim=10)
pca_test_X, e_pca_test_X = pca(test_X, dim=10)
# apply Fishers Linear Discriminant to project PCA trained data
pca_mnist_fld = FLD(pca_train_X, train_y, dim=1)
# fit the FLD process
pca_mnist_fld.fit()
# compute training accuracy
train_acc = pca_mnist_fld.train_accuracy()
# compute test accuracy
test_acc = pca_mnist_fld.test_accuracy(pca_test_X, test_y)
# Display training and test accuracy
print(f'Training accuracy : {train_acc}')
print(f'Test accuracy : {test_acc}')
clf = LinearDiscriminantAnalysis()
# print(pca_train_X.T.shape, train_y.T.flatten().shape) # (400, 10) (400,)
clf.fit(pca_train_X.T, train_y.T.flatten())
print('From sklearn')
print(
'Training accuracy : ',
clf.fit(pca_train_X.T,
train_y.T.flatten()).score(pca_train_X.T, train_y.T.flatten()))
print('Test accuracy : ', clf.score(pca_test_X.T, test_y.T.flatten()))
def main():
digit_range = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
X_train, _, X_test , _ = \
mnist(noTrSamples=6000, noTsSamples=1000, \
digit_range=digit_range, \
noTrPerClass=600, noTsPerClass=100)
X_noisytrain = noisyMethod1(X_train)
# X_noisytrain = noisyMethod2(X_train, 10)
# X_noisytrain = X_noisytrain.T
# X_train = X_train.T
num_epochs = 1000
learning_rate = 0.01
batch_size = 256
net_dims = [784, 1000, 784]
parameters, costs = encoder_decoder_twolayer(X_noisytrain, X_train,
net_dims, num_epochs,
batch_size, learning_rate)
l1, _ = sigmoid(np.dot(parameters["W1"], X_train) + parameters["b1"])
Image, _ = sigmoid(np.dot(parameters["W2"], l1) + parameters["b2"])
fig, axes = plt.subplots(3, 5)
for i in range(5):
axes[0, i].imshow(X_train[:, i].reshape((28, 28)), cmap="gray")
axes[1, i].imshow(X_noisytrain[:, i].reshape((28, 28)), cmap='gray')
axes[2, i].imshow(Image[:, i].reshape((28, 28)), cmap="gray")
plt.savefig("Original_Noisy_Reconstructed_Images.png")
plt.show()
def main():
# getting the subset dataset from MNIST
# binary classification for digits 1 and 7
digit_range = [1, 7]
train_data, train_label, test_data, test_label = \
mnist(noTrSamples=2400,noTsSamples=1000,\
digit_range=digit_range,\
noTrPerClass=1200, noTsPerClass=500)
#convert to binary labels
train_label[train_label == digit_range[0]] = 0
train_label[train_label == digit_range[1]] = 1
test_label[test_label == digit_range[0]] = 0
test_label[test_label == digit_range[1]] = 1
n_in, m = train_data.shape
n_fin = 1
n_h = 500
net_dims = [n_in, n_h, n_fin]
# initialize learning rate and num_iterations
learning_rate = 0.1
num_iterations = 1000
costs, parameters = two_layer_network(train_data, train_label, net_dims, \
num_iterations=num_iterations, learning_rate=learning_rate)
# compute the accuracy for training set and testing set
train_Pred = classify(train_data, parameters)
test_Pred = classify(test_data, parameters)
trAcc = (np.sum(train_Pred == train_label) / train_data.shape[1]) * 100
teAcc = (np.sum(test_Pred == test_label) / test_data.shape[1]) * 100
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
def main():
n_in, m = 784,784
n_h1 = 500
n_h2 = 100
net_dims = [n_in, n_h1, n_h2]
#net_dims = ast.literal_eval( sys.argv[1] )
net_dims.append(10) # Adding the digits layer with dimensionality = 10
print("Network dimensions are:" + str(net_dims))
# getting the subset dataset from MNIST
train_data, train_label, test_data, test_label = \
mnist(noTrSamples=6000,noTsSamples=1000,\
digit_range=[0,1,2,3,4,5,6,7,8,9],\
noTrPerClass=600, noTsPerClass=100)
learning_rate_array = [0.0007, 0.0001, 0.00007]
num_iterations = 300
batch_size = len(train_label)
plots = []
for learning_rate in learning_rate_array:
costs, val_loss, parameters = multi_layer_network(train_data, train_label, net_dims, num_iterations=num_iterations, learning_rate=learning_rate, batch_size=batch_size)
# compute the accuracy for training set and testing set
train_Pred = classify(train_data, parameters)
test_Pred = classify(test_data, parameters)
test_error = ((test_Pred[0]!=test_label[0]).sum())/ test_Pred[0].sum()
print("Test error: {0:0.4f} ".format(test_error))
trAcc = train_Pred - train_label
teAcc = test_Pred - test_label
trAcc[trAcc != 0] = 1
teAcc[teAcc != 0] = 1
trAcc = ( 1 - np.count_nonzero(train_Pred - train_label ) / float(train_Pred.shape[1])) * 100
teAcc = ( 1 - np.count_nonzero(test_Pred - test_label ) / float(test_Pred.shape[1]) ) * 100
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
# points = np.arange(0, 300)
# plot_train, = plt.plot(costs[:len(costs)])
# # plots.append(plot_train)
plot_val, = plt.plot(val_loss)
plots.append(plot_val)
plt.xlabel("No Of Iterations")
plt.ylabel("Cost/ val_loss")
plt.title("Loss vs Number of Iterations with varying Learning rates for ADAM")
plt.legend(plots,learning_rate_array)
plt.savefig("Loss plot for ADAM's Momentum")
def main():
# getting the subset dataset from MNIST
# binary classification for digits 1 and 7
validation_costs = []
test_accuracies = []
# getting the subset dataset from MNIST
data, data_label, test_data, test_label = \
mnist(noTrSamples=60000,noTsSamples=500,\
digit_range=[0,1,2,3,4,5,6,7,8,9],\
noTrPerClass=6000, noTsPerClass=50)
print(test_label)
# initialize learning rate and num_iterations
print(data_label)
num_iterations = 1000
n_in, m = data.shape
n_fin = 784
#n_h = [900,1000,1100,1200,2000]
n_h = [1000]
count = 1
for h in n_h:
print("Hidden layer", h)
net_dims = [n_in, h, n_fin]
# initialize learning rate and num_iterations
learning_rate = 0.1
#fig = plt.figure(figsize=(15, 10))
costs, parameters = two_layer_network(data, data_label,net_dims, \
num_iterations=num_iterations, learning_rate=learning_rate)
sigma = [0.1, 0.5, 1, 5, 10, 15]
costs = []
for sig in sigma:
YPred = classify(noise_induce(test_data, sig), parameters)
cost_pred = reconstruction_loss(YPred, test_data)
print("Cost prediction", cost_pred)
count = 1
costs.append(cost_pred)
for s in range(10):
ind = np.argwhere(test_label[0] == s)[0][0]
print(ind, test_label[0][ind])
img = (YPred.T[ind]).reshape(28, 28)
plt.subplot(5, 2, count)
count += 1
plt.xticks([])
plt.yticks([])
plt.grid(False)
plt.imshow(img, cmap=plt.cm.binary)
plt.show()
print(costs)
def main():
'''
Trains a multilayer network for MNIST digit classification (all 10 digits)
To create a network with 1 hidden layer of dimensions 800
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800]"
The network will have the dimensions [784,800,10]
784 is the input size of digit images (28pix x 28pix = 784)
10 is the number of digits
To create a network with 2 hidden layers of dimensions 800 and 500
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800,500]"
The network will have the dimensions [784,800,500,10]
784 is the input size of digit images (28pix x 28pix = 784)
10 is the number of digits
'''
# net_dims = ast.literal_eval(sys.argv[1])
net_dims = [784,800,500,300]
net_dims.append(10) # Adding the digits layer with dimensionality = 10
print("Network dimensions are:" + str(net_dims))
# getting the subset dataset from MNIST
train_data, train_label, test_data, test_label = \
mnist(noTrSamples=6000,noTsSamples=1000,digit_range=[0,10],\
noTrPerClass=600, noTsPerClass=100)
# initialize learning rate and num_iterations
learning_rate = 0.2
num_iterations = 500
costs, parameters = multi_layer_network(train_data, train_label, net_dims, \
num_iterations=num_iterations, learning_rate=learning_rate)
# compute the accuracy for training set and testing set
train_Pred = classify(train_data, parameters)
test_Pred = classify(test_data, parameters)
trAcc = (1 - np.count_nonzero(train_Pred - train_label) / float(train_Pred.shape[1])) * 100
teAcc = (1 - np.count_nonzero(test_Pred - test_label) / float(test_Pred.shape[1])) * 100
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
### CODE HERE to plot costs
X = range(0, num_iterations, 10)
plt.plot(X, costs)
plt.xlabel("Iterations")
plt.ylabel("Cost")
plt.show()
def main():
# load the data from external load_mnist.py script
train_X, train_y, test_X, test_y = mnist(noTrSamples=400,
noTsSamples=100,
digit_range=[5, 8],
noTrPerClass=200,
noTsPerClass=50)
# perform PCA on test and train data
# Also get the eigen vectors for reconstruction purpose
pca_train_X, e_pca_train_X = pca(train_X, dim=10)
pca_test_X, e_pca_test_X = pca(test_X, dim=10)
# plot the covariance matrix of PCA transformed train data
plot_covariance_matrix(pca_train_X)
# reconstruct images from PCA transformed train data and eigen vectors matrix
recon_train_X = reconstruct_data(pca_train_X, e_pca_train_X)
# show the reconstructed image samples
show_original_and_recon(train_X, recon_train_X, no_of_samples=5)
def main():
trainDevX, trainDevY, testX, testY = \
mnist(noTrSamples=50000,noTsSamples=5000,\
digit_range=[0,1,2,3,4,5,6,7,8,9],\
noTrPerClass=5000, noTsPerClass=500)
perm = np.random.permutation(trainDevX.shape[1])
trainDevX = trainDevX[:, perm]
trainDevY = trainDevY[:, perm]
devX = trainDevX[:, 45000:50000]
devY = trainDevY[:, 45000:50000]
trainX = trainDevX[:, 0:45000]
trainY = trainDevY[:, 0:45000]
#trainY = one_hot(trainY.astype(int), 10)
#devY = one_hot(devY.astype(int), 10)
#testY = one_hot(testY.astype(int), 10)
lDim = [784, 500, 10]
miniBatchSize = 100
k = [[50], [100]]
nEpoch = 200
alpha = 0.001
costHL = []
accTest = []
errDev = []
errTest = []
for i in range(len(k)):
print(k[i])
parameters, costTrain, costDev, t = neuralNetwork(
trainX, trainY, lDim, nEpoch, alpha, devX, devY, k[i],
miniBatchSize)
costHL.append((costTrain, costDev))
errDev.append(costDev[nEpoch - 1])
Ypred, _ = predictClass(devX, parameters)
accDev.append(accuracy(devY, Ypred))
Ypred, AL = predictClass(testX, parameters)
accTest.append(accuracy(testY, Ypred))
errTest.append(costNN(AL, one_hot(testY.astype(int), 10)))
def main():
net_dims = ast.literal_eval( sys.argv[1] )
net_dims.append(10)
print("Network dimensions are:" + str(net_dims))
train_data, train_label, test_data, test_label = \
mnist(noTrSamples=50000,noTsSamples=5000,\
digit_range=[0,1,2,3,4,5,6,7,8,9],\
noTrPerClass=5000, noTsPerClass=500)
learning_rate = 0.001
num_iterations = 500
mini_batch_size = 512
gamma = 0.9
beta = 0.999
NAG_coeff = 0.999
algorithm = ['classical', 'momentum', 'rmsprop', 'adam', 'NAG']
print(algorithm[4])
train_costs, parameters = multi_layer_network(train_data, train_label, net_dims, algorithm[4], num_iterations, learning_rate, gamma, beta, NAG_coeff, mini_batch_size)
train_pred = classify(train_data, parameters)
test_pred = classify(test_data, parameters)
trAcc = (np.count_nonzero(train_pred == train_label) / train_data.shape[1]) * 100
teAcc = (np.count_nonzero(test_pred == test_label) / test_data.shape[1]) * 100
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
print("Cost for training set is {0:0.3f} ".format(train_costs[len(train_costs)-1]))
plt.title("training costs; learning rate="+str(learning_rate))
plt.xlabel("iterations")
plt.ylabel("cost")
plt.plot(list(range(len(train_costs))), train_costs, label='training cost')
plt.legend()
plt.show()
def main():
net_dims = ast.literal_eval(sys.argv[1])
net_dims.append(10) # Adding the digits layer with dimensionality = 10
print("Network dimensions are:" + str(net_dims))
# getting the subset dataset from MNIST
Original_train_data, Original_train_label, test_data, test_label = \
mnist(noTrSamples=60000,noTsSamples=10000,\
digit_range=[0,1,2,3,4,5,6,7,8,9],\
noTrPerClass=6000, noTsPerClass=1000)
train_data = Original_train_data[:, :5000]
train_label = Original_train_label[:, :5000]
validation_data = Original_train_data[:, 5000:6000]
validation_label = Original_train_label[:, 5000:6000]
for i in range(6000, 60000, 6000):
train_data = np.hstack((train_data, Original_train_data[:,
i:i + 5000]))
train_label = np.hstack(
(train_label, Original_train_label[:, i:i + 5000]))
validation_data = np.hstack(
(validation_data, Original_train_data[:, i + 5000:i + 6000]))
validation_label = np.hstack(
(validation_label, Original_train_label[:, i + 5000:i + 6000]))
print(train_data.shape)
print(validation_data.shape)
mini_batches_train = []
mini_batches_val = []
for j in range(10):
x = get_mini_batches(train_data, train_label, 500)
for i in range(len(x)):
mini_batches_train.append(x[i])
y = get_mini_batches(validation_data, validation_label, 100)
for i in range(len(y)):
mini_batches_val.append(y[i])
print(len(mini_batches_train))
print(len(mini_batches_val))
learning_rate = 0.1
num_iterations = 100
num_iter = 1
minibatchsize_train = 500
costs = []
valid_costs = []
parameters = initialize_multilayer_weights(net_dims)
for i in range(len(mini_batches_train)):
loss, parameters, v_loss = multi_layer_network(mini_batches_train[i][0],mini_batches_train[i][1],mini_batches_val[i][0],mini_batches_val[i][1],parameters,net_dims, \
num_iterations=num_iterations, learning_rate=learning_rate,decay_rate=0.01)
#parameters=parameters2.copy()
#parameters2.clear()
if i % 10 == 0:
costs.append(loss)
valid_costs.append(v_loss)
with open('output_nomomentum.txt', 'w') as f:
for item in valid_costs:
f.write("%s\n" % item)
min_train_loss = min(costs)
print("min_train_loss", min_train_loss)
train_Pred = classify(train_data, train_label, parameters)
test_Pred = classify(test_data, test_label, parameters)
valid_Pred = classify(validation_data, validation_label, parameters)
trAcc = 100 * (float(np.sum(train_Pred == train_label)) /
train_label.shape[1])
teAcc = 100 * (float(np.sum(test_Pred == test_label)) /
test_label.shape[1])
VaAcc = 100 * (float(np.sum(valid_Pred == validation_label)) /
validation_label.shape[1])
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
print("Accuracy for Validation set is {0:0.3f} %".format(VaAcc))
### CODE HERE to plot costs
plt.plot(costs)
#plt.show()
plt.plot(valid_costs)
plt.ylabel('costs')
plt.xlabel('iterations (multiples of 10)')
plt.title(
"Validation and training costs vs iterations at learning rate 0.2")
plt.show()
print("Test Error:", 1 - (teAcc / 100))
def main():
start_time = time.time()
train_data, train_label, test_data, test_label = mnist()
noise = 0.4
testnoise1 = 0.1
testnoise2 = 0.2
testnoise3 = 0.3
testnoise4 = 0.4
trX_noisy = get_corrupted_input(train_data, noise)
tsX_noisy1 = get_corrupted_input(test_data, testnoise1)
tsX_noisy2 = get_corrupted_input(test_data, testnoise2)
tsX_noisy3 = get_corrupted_input(test_data, testnoise3)
tsX_noisy4 = get_corrupted_input(test_data, testnoise4)
n_in, m = train_data.shape
n_fin, m = train_data.shape
n_h = 1000
net_dims = [n_in, n_h, n_fin]
# initialize learning rate and num_iterations
learning_rate = 1
num_iterations = 1000
costs, parameters = two_layer_network(trX_noisy,
train_data,
net_dims,
num_iterations=num_iterations,
learning_rate=learning_rate)
fig = plt.figure()
plt.imshow(test_data[:, 20].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Test_Sample with gaussian noise" + str(noise) + "is")
plt.show()
fig.savefig("Test_Sample with gaussian noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(test_data[:, 80].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Test_Sample with gaussian noise" + str(noise) + "is")
plt.show()
fig.savefig("Test_Sample 8with gaussian noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(test_data[:, 91].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Test_Sample with gaussian noise" + str(noise) + "is")
plt.show()
fig.savefig("Test_Sample 9 with gaussian noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy1[:, 20].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Noisy_Test_Sample..with_gaussian_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Samplewith_gaussian_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy2[:, 20].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Noisy_Test_Sample withgaussiannoise" + str(testnoise2) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample with_gaussiannoise" + str(testnoise2) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy3[:, 20].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Noisy_Test_Sample with_gaussian_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample with_gaussian_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy4[:, 20].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Noisy_Test_Sample with_gaussian_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample with_gaussian_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
fig = plt.figure()
plt.imshow(tsX_noisy1[:, 80].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Noisy_Test_Sample 8with_gaussian_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample8with_gaussian_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy2[:, 80].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Noisy_Test_Sample 8 with_gaussian_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample8 with_gaussian_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy3[:, 80].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Noisy_Test_Sample 8with_gaussian_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample 8with_gaussian_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy4[:, 80].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Noisy_Test_Sample 8with_gaussian_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample8 with_gaussian_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
fig = plt.figure()
plt.imshow(tsX_noisy1[:, 91].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Noisy_Test_Sample 9with_gaussian_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample8with_gaussian_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy2[:, 91].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Noisy_Test_Sample 9 with_gaussian_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample9 with_gaussian_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy3[:, 91].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Noisy_Test_Sample 9with_gaussian_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample 9 with_gaussian_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy4[:, 91].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Noisy_Test_Sample 9 with_gaussian_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample 9 with_gaussian_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is.png")
test_Pred1 = denoise(tsX_noisy1, parameters)
test_Pred2 = denoise(tsX_noisy2, parameters)
test_Pred3 = denoise(tsX_noisy3, parameters)
test_Pred4 = denoise(tsX_noisy4, parameters)
print("accuracy of test sample 1 with gaussian noise" + str(testnoise1) +
"is " + str((1 - error(test_Pred1, test_data)) * 100) + "%")
print("accuracy of test sample 2 with gaussian noise" + str(testnoise2) +
"is " + str((1 - error(test_Pred2, test_data)) * 100) + "%")
print("accuracy of test sample 3 with gaussian noise" + str(testnoise3) +
"is " + str((1 - error(test_Pred3, test_data)) * 100) + "%")
print("accuracy of test sample 4 with gaussian noise" + str(testnoise4) +
"is " + str((1 - error(test_Pred4, test_data)) * 100) + "%")
fig = plt.figure()
plt.imshow(test_Pred1[:, 20].reshape(28, -1), cmap=plt.cm.binary)
plt.title("denoised_Sample with_gaussian_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random noise_gaussian_Test_Sample op1")
fig = plt.figure()
plt.imshow(test_Pred2[:, 20].reshape(28, -1), cmap=plt.cm.binary)
plt.title("denoised_Sample with_gaussian_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_gaussian noiseTest_Sample op2")
fig = plt.figure()
plt.imshow(test_Pred3[:, 20].reshape(28, -1), cmap=plt.cm.binary)
plt.title("denoised_Sample with_gaussian_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_gaussian_noiseTest_Sample op3")
fig = plt.figure()
plt.imshow(test_Pred4[:, 20].reshape(28, -1), cmap=plt.cm.binary)
plt.title("denoised_Sample with_gaussian_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_gaussian noiseTest_Sample op4")
fig = plt.figure()
plt.imshow(test_Pred1[:, 80].reshape(28, -1), cmap=plt.cm.binary)
plt.title("denoised_Sample 8 with_gaussian_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random noise_gaussian_Test_Sample8 op1")
fig = plt.figure()
plt.imshow(test_Pred2[:, 80].reshape(28, -1), cmap=plt.cm.binary)
plt.title("denoised_Sample with_gaussian_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random noiseTest_Sample 8 op2")
fig = plt.figure()
plt.imshow(test_Pred3[:, 80].reshape(28, -1), cmap=plt.cm.binary)
plt.title("denoised_Sample with_gaussian_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_gaussian_noiseTest_Sample 8 op3")
fig = plt.figure()
plt.imshow(test_Pred4[:, 80].reshape(28, -1), cmap=plt.cm.binary)
plt.title("denoised_Sample with_gaussian_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_gaussian noiseTest_Sample 8 op4")
fig = plt.figure()
plt.imshow(test_Pred1[:, 91].reshape(28, -1), cmap=plt.cm.binary)
plt.title("denoised_Sample 9 with_gaussian_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random noise_gaussian_Test_Sample9 op1")
fig = plt.figure()
plt.imshow(test_Pred2[:, 91].reshape(28, -1), cmap=plt.cm.binary)
plt.title("denoised_Sample 9 with_random_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_gaussian noiseTest_Sample 9 op2")
fig = plt.figure()
plt.imshow(test_Pred3[:, 91].reshape(28, -1), cmap=plt.cm.binary)
plt.title("denoised_Sample 9 with_gaussian_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_gaussian_noiseTest_Sample 9 op3")
fig = plt.figure()
plt.imshow(test_Pred4[:, 91].reshape(28, -1), cmap=plt.cm.binary)
plt.title("denoised_Sample 9 with_gaussian_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_gaussian noiseTest_Sample 9 op4")
plt.plot(costs, label='Training cost')
plt.title("training cost with learning rate =" + str(learning_rate) +
"iterations" + str(num_iterations) + "noise :" + str(noise))
plt.show()
fig.savefig("training cost")
print("Total execution time: %s minutes!!" %
((time.time() - start_time) // 60))
def main():
start_time = time.time()
train_data, train_label, test_data, test_label = mnist()
# print(param)
n_rows = train_data.shape[0]
n_cols = train_data.shape[1]
test_rows = test_data.shape[0]
test_cols = test_label.shape[1]
noise = 5
testnoise1 = 4
testnoise2 = 5
testnoise3 = 6
testnoise4 = 7
trX_noisy = masking_noise(train_data, noise)
tsX_noisy1 = masking_noise(test_data, testnoise1)
tsX_noisy2 = masking_noise(test_data, testnoise2)
tsX_noisy3 = masking_noise(test_data, testnoise3)
tsX_noisy4 = masking_noise(test_data, testnoise4)
n_in, m = train_data.shape
n_fin, m = train_data.shape
n_h = 1000
net_dims = [n_in, n_h, n_fin]
# initialize learning rate and num_iterations
learning_rate = 1
num_iterations = 1000
costs, parameters = two_layer_network(trX_noisy,
train_data,
net_dims,
num_iterations=num_iterations,
learning_rate=learning_rate)
fig = plt.figure()
plt.imshow(test_data[:, 99].reshape(28, -1), cmap="gray")
plt.title("Test_Sample with random noise" + str(noise) + "is")
plt.show()
fig.savefig("Test_Sample with random noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(test_data[:, 72].reshape(28, -1), cmap="gray")
plt.title("Test_Sample with random noise" + str(noise) + "is")
plt.show()
fig.savefig("Test_Sample 8with random noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(test_data[:, 20].reshape(28, -1), cmap="gray")
plt.title("Test_Sample with random noise" + str(noise) + "is")
plt.show()
fig.savefig("Test_Sample 9 with random noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy1[:, 99].reshape(28, -1), cmap="gray")
plt.title("Noisy_Test_Sample..with_random_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Samplewith_random_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy2[:, 99].reshape(28, -1), cmap="gray")
plt.title("Noisy_Test_Sample with_random_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample with_random_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy3[:, 99].reshape(28, -1), cmap="gray")
plt.title("Noisy_Test_Sample with_random_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample with_random_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy4[:, 99].reshape(28, -1), cmap="gray")
plt.title("Noisy_Test_Sample with_random_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample with_random_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
fig = plt.figure()
plt.imshow(tsX_noisy1[:, 72].reshape(28, -1), cmap="gray")
plt.title("Noisy_Test_Sample 8with_random_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample8with_random_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy2[:, 72].reshape(28, -1), cmap="gray")
plt.title("Noisy_Test_Sample 8 with_random_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample8 with_random_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy3[:, 72].reshape(28, -1), cmap="gray")
plt.title("Noisy_Test_Sample 8with_random_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample 8with_random_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy4[:, 72].reshape(28, -1), cmap="gray")
plt.title("Noisy_Test_Sample 8with_random_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample8 with_random_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
fig = plt.figure()
plt.imshow(tsX_noisy1[:, 20].reshape(28, -1), cmap="gray")
plt.title("Noisy_Test_Sample 9with_random_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample8with_random_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy2[:, 20].reshape(28, -1), cmap="gray")
plt.title("Noisy_Test_Sample 9 with_random_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample9 with_random_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy3[:, 20].reshape(28, -1), cmap="gray")
plt.title("Noisy_Test_Sample 9with_random_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample 9 with_random_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is.png")
fig = plt.figure()
plt.imshow(tsX_noisy4[:, 20].reshape(28, -1), cmap="gray")
plt.title("Noisy_Test_Sample 9 with_random_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Noisy_Test_Sample 9 with_random_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is.png")
test_Pred1 = denoise(tsX_noisy1, parameters)
test_Pred2 = denoise(tsX_noisy2, parameters)
test_Pred3 = denoise(tsX_noisy3, parameters)
test_Pred4 = denoise(tsX_noisy4, parameters)
print("accuracy of test sample 2 with random noise" + str(testnoise1) +
"is " + str((1 - error(test_Pred1, test_data)) * 100) + "%")
print("accuracy of test sample 2 with random noise" + str(testnoise2) +
"is " + str((1 - error(test_Pred2, test_data)) * 100) + "%")
print("accuracy of test sample 2 with random noise" + str(testnoise3) +
"is " + str((1 - error(test_Pred3, test_data)) * 100) + "%")
print("accuracy of test sample 2 with random noise" + str(testnoise4) +
"is " + str((1 - error(test_Pred4, test_data)) * 100) + "%")
fig = plt.figure()
plt.imshow(test_Pred1[:, 99].reshape(28, -1), cmap="gray")
plt.title("denoised_Sample with_random_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random noise_random_Test_Sample op1")
fig = plt.figure()
plt.imshow(test_Pred2[:, 99].reshape(28, -1), cmap="gray")
plt.title("denoised_Sample with_random_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random noiseTest_Sample op2")
fig = plt.figure()
plt.imshow(test_Pred3[:, 99].reshape(28, -1), cmap="gray")
plt.title("denoised_Sample with_random_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random_noiseTest_Sample op3")
fig = plt.figure()
plt.imshow(test_Pred4[:, 99].reshape(28, -1), cmap="gray")
plt.title("denoised_Sample with_random_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random noiseTest_Sample op4")
fig = plt.figure()
plt.imshow(test_Pred1[:, 72].reshape(28, -1), cmap="gray")
plt.title("denoised_Sample 8 with_random_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random noise_random_Test_Sample8 op1")
fig = plt.figure()
plt.imshow(test_Pred2[:, 72].reshape(28, -1), cmap="gray")
plt.title("denoised_Sample with_random_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random noiseTest_Sample 8 op2")
fig = plt.figure()
plt.imshow(test_Pred3[:, 72].reshape(28, -1), cmap="gray")
plt.title("denoised_Sample with_random_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random_noiseTest_Sample 8 op3")
fig = plt.figure()
plt.imshow(test_Pred4[:, 72].reshape(28, -1), cmap="gray")
plt.title("denoised_Sample with_random_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random noiseTest_Sample 8 op4")
fig = plt.figure()
plt.imshow(test_Pred1[:, 20].reshape(28, -1), "gray")
plt.title("denoised_Sample 9 with_random_noise" + str(testnoise1) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random noise_random_Test_Sample9 op1")
fig = plt.figure()
plt.imshow(test_Pred2[:, 20].reshape(28, -1), cmap="gray")
plt.title("denoised_Sample 9 with_random_noise" + str(testnoise2) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random noiseTest_Sample 9 op2")
fig = plt.figure()
plt.imshow(test_Pred3[:, 20].reshape(28, -1), cmap="gray")
plt.title("denoised_Sample 9 with_random_noise" + str(testnoise3) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random_noiseTest_Sample 9 op3")
fig = plt.figure()
plt.imshow(test_Pred4[:, 20].reshape(28, -1), cmap="gray")
plt.title("denoised_Sample 9 with_random_noise" + str(testnoise4) +
"and train noise" + str(noise) + "is")
plt.show()
fig.savefig("Denoised_random noiseTest_Sample 9 op4")
plt.plot(costs, label='Training cost')
plt.title("training cost with learning rate =" + str(learning_rate) +
"iterations" + str(num_iterations) + "noise :" + str(noise))
plt.show()
fig.savefig("trainingCost")
print("Total execution time: %s minutes!!" %
((time.time() - start_time) // 60))
def main():
# getting the subset dataset from MNIST
# binary classification for digits 1 and 7
validation_costs = []
test_accuracies = []
# getting the subset dataset from MNIST
data, data_label, test_data, test_label = \
mnist(noTrSamples=6000,noTsSamples=50,\
digit_range=[0,1,2,3,4,5,6,7,8,9],\
noTrPerClass=600, noTsPerClass=5)
if par == 1:
# initialize learning rate and num_iterations
num_iterations = 3000
n_in, m = data.shape
learning_rate = 0.2
net_dims = [784, 500, 784]
costs_1, parameters_1, A_1 = two_layer_network(data, data_label,net_dims, \
num_iterations=num_iterations, learning_rate=learning_rate)
learning_rate = 0.4
num_iterations = 1500
net_dims = [500, 200, 500]
costs_2,parameters_2, A_2 = two_layer_network(A_1, data_label,net_dims, \
num_iterations=num_iterations, learning_rate=learning_rate)
net_dims = [200, 100, 200]
costs_3, parameters_3, A_3 = two_layer_network(A_2, data_label,net_dims, \
num_iterations=num_iterations, learning_rate=learning_rate)
parameters = {}
parameters["W1"] = parameters_1["W1"]
parameters["b1"] = parameters_1["b1"]
parameters["W2"] = parameters_2["W1"]
parameters["b2"] = parameters_2["b1"]
parameters["W3"] = parameters_3["W1"]
parameters["b3"] = parameters_3["b1"]
print(parameters["W1"].shape)
print(parameters["W2"].shape)
#print(parameters["W3"].shape)
print(parameters)
net_dims = [784, 500, 200, 100, 10]
learning_rate = 1
num_iterations = 5000
costs_4, parameters_4 = multi_layer_network(parameters, data_last, data_last_label, net_dims, \
num_iterations=num_iterations, learning_rate=learning_rate)
train_Pred = deepMultiClassNetwork_starter.classify(data, parameters_4)
test_Pred = deepMultiClassNetwork_starter.classify(
test_data, parameters_4)
print("shape-DATA", data_label.shape)
print("shape-TRAIN", train_Pred.shape)
print("count:", np.count_nonzero(train_Pred - data_label))
trAcc = (1 - np.count_nonzero(train_Pred - data_label) /
float(train_Pred.shape[1])) * 100
teAcc = (1 - np.count_nonzero(test_Pred - test_label) /
float(test_Pred.shape[1])) * 100
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
if par == 2:
clf = svm.SVC(gamma='scale', decision_function_shape='ovo')
clf.fit(data.T, data_label.T)
out = clf.predict(test_data.T)
print(out)
print("Accuracy",
(np.sum(out == test_label) / test_label.shape[1]) * 100)
if par == 3:
clf = LogisticRegression(random_state=0,
solver='newton-cg',
multi_class='multinomial').fit(
data.T, data_label.T)
score = clf.score(test_data.T, test_label.T)
print(score)
if par == 4:
clf = GaussianNB()
clf.fit(data.T, data_label.T)
score = clf.score(test_data.T, test_label.T)
print(score)
clf = BernoulliNB()
clf.fit(data.T, data_label.T)
score = clf.score(test_data.T, test_label.T)
print(score)
clf = MultinomialNB()
clf.fit(data.T, data_label.T)
score = clf.score(test_data.T, test_label.T)
print(score)
if par == 5:
clf = DecisionTreeClassifier(criterion='gini', random_state=0)
clf.fit(data.T, data_label.T)
score = clf.score(test_data.T, test_label.T)
clf = RandomForestClassifier(n_estimators=100,
max_depth=2,
random_state=0)
clf.fit(data.T, np.ravel(data_label.T))
score = clf.score(test_data.T, np.ravel(test_label.T))
print(score)
def main():
'''
Trains a multilayer network for MNIST digit classification (all 10 digits)
To create a network with 1 hidden layer of dimensions 800
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800]"
The network will have the dimensions [784,800,10]
784 is the input size of digit images (28pix x 28pix = 784)
10 is the number of digits
To create a network with 2 hidden layers of dimensions 800 and 500
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800,500]"
The network will have the dimensions [784,800,500,10]
784 is the input size of digit images (28pix x 28pix = 784)
10 is the number of digits
'''
net_dims = ast.literal_eval(sys.argv[1])
net_dims.append(10) # Adding the digits layer with dimensionality = 10
print("Network dimensions are:" + str(net_dims))
# getting the subset dataset from MNIST
train_data, train_label, test_data, test_label = \
mnist(noTrSamples=60000,noTsSamples=10000,\
digit_range=[0,1,2,3,4,5,6,7,8,9],\
noTrPerClass=6000, noTsPerClass=1000)
train_data, validation_data, train_label, validation_label = train_test_split(
train_data.T,
train_label.T,
test_size=10000,
train_size=50000,
random_state=42)
learning_rate = 0.2
num_iterations = 1
epochs = 10
parameters = initialize_multilayer_weights(net_dims)
process = psutil.Process(os.getpid())
total_time = 0
for j in range(0, epochs):
start = time.time()
for i in range(0, 10):
print "EPOCH----------------------------->=", j
print "BATCH----------------------------->=", i
mini_train_data = train_data[i * 5000:(i * 5000) + 5000]
mini_train_label = train_label[i * 5000:(i * 5000) + 5000]
mini_train_data = mini_train_data.T
mini_train_label = mini_train_label.T
costs,Vcosts,parameters = multi_layer_network(i,parameters,mini_train_data,mini_train_label,validation_data.T,validation_label.T, net_dims, \
num_iterations=num_iterations, learning_rate=learning_rate)
timetaken = time.time() - start
total_time = total_time + timetaken
print "TIME TAKEN=>", timetaken, " seconds"
print(process.memory_info().rss)
times.append(timetaken)
print "TOTAL TIME TAKEN=", total_time
# compute the accuracy for training set and testing set
train_Pred = classify(train_data.T, parameters)
test_Pred = classify(test_data, parameters)
trAcc = calculateAccuracy(train_Pred.T, train_label.T)
teAcc = calculateAccuracy(test_Pred.T, test_label)
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
print "MEMORY USAGE FOR TOTAL EPOCHS----->", (process.memory_info().rss *
0.001) / 784, "MB"
## CODE HERE to plot costs
#train error vs iterations here
x = []
for i in range(0, epochs):
x.append(i)
print costs
print Vcosts
print x
plt.plot(x, costs) #,'ro')
plt.plot(x, Vcosts) #,'b^')
plt.show()
plt.plot(x, times)
plt.show()
def main():
net_dims = [784, 500, 100, 10]
train_data, train_label, test_data, test_label, valid_data, valid_label = \
mnist(noTrSamples=5000, noTsSamples=1000,
digit_range=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
noTrPerClass=500, noTsPerClass=100, noVdSamples=1000, noVdPerClass=100)
num_iterations_list = [100, 500, 1000]
# Add gradient method and its corresponding learning rate to the array.
gradient_methods = [NO_MOMENTUM, MOMENTUM, NAG, RMSPROP, ADAM]
learning_rates = [0.1, 0.05, 0.01, 0.001]
for num_iterations in num_iterations_list:
print(f"Running {num_iterations} Iterations")
for learning_rate in learning_rates:
all_costs = []
all_vcosts = []
time_per_algo = []
training_accuracy_per_algo = []
testing_accuracy_per_algo = []
for gradient_method in gradient_methods:
start_time = datetime.datetime.now()
mm = MultiLayerNeuralNetwork(net_dims, learning_rate,
num_iterations, gradient_method)
mm.fit(train_data, train_label, valid_data, valid_label)
end_time = datetime.datetime.now()
# compute the accuracy for training set and testing set
Y_one_hot = one_hot(train_label, 10)
t_one_hot = one_hot(test_label, 10)
train_Pred = mm.predict(train_data, Y_one_hot)
test_Pred = mm.predict(test_data, t_one_hot)
trAcc = np.count_nonzero(
train_Pred == train_label) / train_label.shape[1] * 100
teAcc = np.count_nonzero(
test_Pred == test_label) / test_label.shape[1] * 100
print(f"Accuracy for training set is {trAcc:0.03f} %")
print(f"Accuracy for testing set is {teAcc:0.03f} %")
print(
f"Time for the algo - {gradient_method} for {num_iterations} iterations was {(end_time - start_time).total_seconds()}"
)
print("------------------------------------------------------")
time_per_algo.append((end_time - start_time).total_seconds())
testing_accuracy_per_algo.append(teAcc)
training_accuracy_per_algo.append(trAcc)
all_vcosts.append(mm.validation_costs)
all_costs.append(mm.costs)
plot_costs_graph(
all_costs, gradient_methods,
f"outputs/brand/allcosts-{num_iterations}-{learning_rate}.png",
learning_rate, False)
plot_costs_graph(
all_vcosts, gradient_methods,
f"outputs/brand/allValidcosts-{num_iterations}-{learning_rate}.png",
learning_rate, True)
print(time_per_algo)
print(testing_accuracy_per_algo)
print(training_accuracy_per_algo)
plot_bar_graph(
time_per_algo, gradient_methods,
f"outputs/brand/time-{num_iterations}-{learning_rate}.png",
f"Plot of Time taken when using various algorithms for learning rate {learning_rate}",
"Time")
plot_bar_graph(
testing_accuracy_per_algo, gradient_methods,
f"outputs/brand/testing-accuracy-{num_iterations}-{learning_rate}.png",
f"Plot of Testing Accuracy at LR: {learning_rate}",
"Testing Accuracy")
plot_bar_graph(
training_accuracy_per_algo, gradient_methods,
f"outputs/brand/training-accuracy-{num_iterations}-{learning_rate}.png",
f"Plot of Training Accuracy at LR: {learning_rate}",
"Training Accuracy")
def main():
n_in, m = 784, 784
n_h1 = 500
n_h2 = 100
net_dims = [n_in, n_h1, n_h2]
#net_dims = ast.literal_eval( sys.argv[1] )
net_dims.append(10) # Adding the digits layer with dimensionality = 10
print("Network dimensions are:" + str(net_dims))
# getting the subset dataset from MNIST
train_data, train_label, test_data, test_label = \
mnist(noTrSamples=6000,noTsSamples=1000,\
digit_range=[0,1,2,3,4,5,6,7,8,9],\
noTrPerClass=600, noTsPerClass=100)
# initialize learning rate and num_iterations
learning_rate = [0.00001, 0.0007, 0.0001]
num_iterations = 300
decay_rate = 0.01
# batch_size = 10
batch_size = len(train_label)
for x in learning_rate:
import matplotlib.pyplot as plt
costs, val_loss, parameters = multi_layer_network(
train_data,
train_label,
net_dims,
num_iterations=num_iterations,
learning_rate=x,
decay_rate=decay_rate,
batch_size=batch_size)
# compute the accuracy for training set and testing set
train_Pred = classify(train_data, parameters)
test_Pred = classify(test_data, parameters)
test_error = (
(test_Pred[0] != test_label[0]).sum()) / test_Pred[0].sum()
print("Test error: {0:0.4f} ".format(test_error))
trAcc = train_Pred - train_label
teAcc = test_Pred - test_label
trAcc[trAcc != 0] = 1
teAcc[teAcc != 0] = 1
trAcc = (1 - np.count_nonzero(train_Pred - train_label) /
float(train_Pred.shape[1])) * 100
teAcc = (1 - np.count_nonzero(test_Pred - test_label) /
float(test_Pred.shape[1])) * 100
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
# train_loss, = plt.plot([x for x in range(num_iterations) if x % 10 == 0], costs)
# val_loss, = plt.plot([x for x in range(num_iterations) if x % 10 == 0], val_loss)
points = np.arange(0, 300)
train_loss, = plt.plot(points, costs[:300])
val_loss, = plt.plot(val_loss)
plt.xlabel("No Of Iterations")
plt.ylabel("Cost/ val_loss")
plt.title("Loss vs Number of Iterations with " + str(x) +
" Learning rate")
plt.legend((train_loss, val_loss),
("Train set Loss", "Validation Set Loss"))
plt.show()
def main():
'''
Trains a multilayer network for MNIST digit classification (all 10 digits)
To create a network with 1 hidden layer of dimensions 800
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800]"
The network will have the dimensions [784,800,10]
784 is the input size of digit images (28pix x 28pix = 784)
10 is the number of digits
To create a network with 2 hidden layers of dimensions 800 and 500
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800,500]"
The network will have the dimensions [784,800,500,10]
784 is the input size of digit images (28pix x 28pix = 784)
10 is the number of digits
'''
n_input, m = 784, 784
n_hidden1 = 500
n_hidden2 = 100
net_dims = [n_input, n_hidden1, n_hidden2]
#net_dims = ast.literal_eval(sys.argv[1])
net_dims.append(10) # Adding the digits layer with dimensionality = 10
print("Network dimensions are:" + str(net_dims))
# getting the subset dataset from Fashion_mnist
orig_train_data, orig_train_label, test_data, test_label = \
mnist(noTrSamples=6000, noTsSamples=1000, digit_range=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], noTrPerClass=600,
noTsPerClass=100)
train_data = orig_train_data[:, :500]
train_label = orig_train_label[:, :500]
validation_data = orig_train_data[:, 500:600]
validation_label = orig_train_label[:, 500:600]
for i in range(600, 6000, 600):
train_data = np.hstack((train_data, orig_train_data[:, i:i + 500]))
train_label = np.hstack((train_label, orig_train_label[:, i:i + 500]))
validation_data = np.hstack(
(validation_data, orig_train_data[:, i + 500:i + 600]))
validation_label = np.hstack(
(validation_label, orig_train_label[:, i + 500:i + 600]))
learning_rate = 0.01
decay_rate = 0.01
num_iterations = 500
batch_size = 10
costs, parameters, vcosts = multi_layer_network(
train_data,
train_label,
validation_data,
validation_label,
net_dims,
num_iterations=num_iterations,
learning_rate=learning_rate,
decay_rate=decay_rate,
batch_size=batch_size)
# compute the accuracy for training set and testing set
train_Pred = classify(train_data, parameters)
test_Pred = classify(test_data, parameters)
val_Pred = classify(validation_data, parameters)
trAcc = (1 - np.count_nonzero(train_Pred - train_label) /
float(train_Pred.shape[1])) * 100
teAcc = (1 - np.count_nonzero(test_Pred - test_label) /
float(test_Pred.shape[1])) * 100
valAcc = (1 - np.count_nonzero(val_Pred - validation_label) /
float(val_Pred.shape[1])) * 100
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
print("Accuracy for validation set is {0:0.3f} %".format(valAcc))
X = np.arange(0, 500)
plt.plot(X, costs, label="Train Cost")
plt.plot(X, vcosts, label="Validation Cost")
plt.xlabel("No Of Iterations")
plt.ylabel("Training Cost / Validation Cost")
plt.title("Cost vs Number of Iterations with " + str(learning_rate) +
" Learning rate")
plt.legend()
plt.show()
def sgd(cost, params, lr=0.05):
grads = T.grad(cost=cost, wrt=params)
updates = []
for p, g in zip(params, grads):
updates.append([p, p - g * lr])
return updates
def model(X, w_h, w_o):
h = T.nnet.sigmoid(T.dot(X, w_h))
pyx = T.nnet.softmax(T.dot(h, w_o))
return pyx
trX, teX, trY, teY = mnist(onehot=True)
X = T.fmatrix()
Y = T.fmatrix()
w_h = init_weights((784, 625))
w_o = init_weights((625, 10))
py_x = model(X, w_h, w_o)
y_x = T.argmax(py_x, axis=1)
cost = T.mean(T.nnet.categorical_crossentropy(py_x, Y))
params = [w_h, w_o]
updates = sgd(cost, params)
train = theano.function(inputs=[X, Y],
def main():
'''
Trains a multilayer network for MNIST digit classification (all 10 digits)
To create a network with 1 hidden layer of dimensions 800
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800]"
The network will have the dimensions [784,800,10]
784 is the input size of digit images (28pix x 28pix = 784)
10 is the number of digits
To create a network with 2 hidden layers of dimensions 800 and 500
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800,500]"
The network will have the dimensions [784,800,500,10]
784 is the input size of digit images (28pix x 28pix = 784)
10 is the number of digits
'''
net_dims = ast.literal_eval(sys.argv[1])
net_dims.append(10) # Adding the digits layer with dimensionality = 10
print("Network dimensions are:" + str(net_dims))
# getting the subset dataset from MNIST
train_data, train_label, test_data, test_label = \
mnist(noTrSamples=6000,noTsSamples=1000,\
digit_range=[0,1,2,3,4,5,6,7,8,9],\
noTrPerClass=600, noTsPerClass=100)
valid_data, valid_label = train_data[:, 5000:6000], train_label[:,
5000:6000]
# initialize learning rate and num_iterations
learning_rate = 10.0
num_iterations = 1000
num_train_samples = 5000
num_validate_samples = 1000
num_test_samples = 1000
costs, parameters ,costs_valid = multi_layer_network(train_data, train_label,valid_data,valid_label, net_dims, \
num_iterations=num_iterations, learning_rate= learning_rate, decay_rate = 0.0)
# compute the accuracy for training set and testing set
train_Pred = classify(train_data, parameters)
test_Pred = classify(test_data, parameters)
#print (train_label.shape)
#print(train_Pred.shape)
#print (train_label)
#print(train_Pred)
train_label_new = one_hot(train_label.astype(int), 10)
test_label_new = one_hot(test_label.astype(int), 10)
result_train = np.sum(abs(train_Pred - train_label_new.T))
print("Train error is : %f" % (result_train))
result_test = np.sum(abs(test_Pred - test_label_new.T))
print("Test error is : %f" % (result_test))
trAcc = 1 / num_train_samples * np.sum(num_train_samples -
result_train) * 100
teAcc = 1 / num_test_samples * np.sum(num_test_samples - result_test) * 100
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
### CODE HERE to plot costs
x = range(0, int(num_iterations / 10))
plt.plot(x, costs)
plt.plot(x, costs_valid)
plt.xlabel('iterations')
plt.ylabel('Costs')
plt.title('Training and validation')
plt.show()
def main():
# getting the subset dataset from MNIST
# binary classification for digits 1 and 7
hiddenNodes = [100, 200, 500]
digit_range = [1, 7]
data, label, test_data, test_label = \
mnist(noTrSamples=2400,noTsSamples=1000,\
digit_range=digit_range,\
noTrPerClass=1200, noTsPerClass=500)
train_data = data[:, 0:1000]
train_data2 = data[:, 1400:2400]
train_data = np.column_stack((train_data, train_data2))
train_label = label[:, 0:1000]
train_label2 = label[:, 1400:2400]
train_label = np.column_stack((train_label, train_label2))
validation_data = data[:, 1000:1400]
validation_label = label[:, 1000:1400]
# intialially its 1 and 7
#convert to binary labels
# 0 -> 1
# 1 -> 7
train_label[train_label == digit_range[0]] = 0
train_label[train_label == digit_range[1]] = 1
test_label[test_label == digit_range[0]] = 0
test_label[test_label == digit_range[1]] = 1
validation_label[validation_label == digit_range[0]] = 0
validation_label[validation_label == digit_range[1]] = 1
mxValidationAccuracy = 0
testError = 0
nodes = 200
mxvcost = 99
fig, axs = plt.subplots(1, 3, constrained_layout=True)
fig.suptitle('\nTraining cost v iterations\n')
i = 0
for n_h in hiddenNodes:
if (n_h == 400):
print "\nVarying hidden nodes to check accuracy"
print "Number of hidden nodes :", n_h
n_in, m = train_data.shape
# n_in == 784 , m==2000
n_fin = 1
# for 500 nodes
# NN -> 784->500_->1
net_dims = [n_in, n_h, n_fin]
# initialize learning rate and num_iterations
learning_rate = 0.1
num_iterations = 1000
# print train_label
costs,vcosts, parameters,A2 = two_layer_network(train_data, train_label, net_dims, \
validation_data,validation_label,num_iterations=num_iterations, learning_rate=learning_rate)
# compute the accuracy for training set and testing set
train_Pred = classify(train_data, parameters)
train_Accuracy = calculateAccuracy(train_label, train_Pred)
validation_Pred = classify(validation_data, parameters)
validation_Accuracy = calculateAccuracy(validation_label,
validation_Pred)
test_Pred = classify(test_data, parameters)
test_Accuracy = calculateAccuracy(test_label, test_Pred)
print "Accuracy for training set is ", train_Accuracy
print "Accuracy for validation set is ", validation_Accuracy
print "Accuracy for testing set is ", test_Accuracy
print "Test Error is ", 100 - test_Accuracy, "%", "\n"
if (float(vcosts[100]) < float(mxvcost)):
mxvcost = vcosts[100]
testError = test_Accuracy
nodes = n_h
mxValidationAccuracy = validation_Accuracy
# CODE HERE TO PLOT costs vs iterations
iterations = [k for k in range(0, 1001, 10)]
axs[i].plot(iterations, costs, 'r', label="Training Cost")
axs[i].plot(iterations, vcosts, 'b', label="Validation Cost")
axs[i].set_title("Hidden nodes :" + str(n_h))
axs[i].set_xlabel('Iterations')
axs[i].set_ylabel('Costs')
axs[i].legend()
i += 1
# axs[1].plot(iterations,vcosts)
# axs[1].set_xlabel('Iterations')
# axs[1].set_title('\nValidation cost vs iterations\n"')
# axs[1].set_ylabel('Costs')
plt.show()
print "Best Validation Cost is: ", mxvcost, " where number of hidden nodes :", nodes
print "Validation accuracy for this architecture :", mxValidationAccuracy
print "Test Accuracy for this architecture :", testError
print "\n"
#coding=utf-8
import tensorflow as tf
from load_mnist import mnist
mnist = mnist()
mnist.load_mnist('../../data', 'train')
iter_num = 200000 #迭代次数
learning_rate = 0.001 #学习率
batch_size = 64 #每次训练采用的样本数
display = 20 #输出的轮数
input_num = 28 * 28 #输入图像的像素大小
class_num = 10 #类别的个数
drop_out = 0.8 #剪枝的比率
X = tf.placeholder(tf.float32, shape=(None, input_num))
y = tf.placeholder(tf.float32, shape=[None, class_num])
keep_prob = tf.placeholder(tf.float32)
def conv(input, w, b, name):
return tf.nn.relu(tf.nn.bias_add(
tf.nn.conv2d(input, w, strides=[1, 1, 1, 1], padding='SAME'), b),
name=name)
def maxpooling(input, k, name):
return tf.nn.max_pool(input,
ksize=[1, k, k, 1],
strides=[1, k, k, 1],
def main():
'''
Trains a multilayer network for MNIST digit classification (all 10 digits)
To create a network with 1 hidden layer of dimensions 800
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800]"
The network will have the dimensions [784,800,10]
784 is the input size of digit images (28pix x 28pix = 784)
10 is the number of digits
To create a network with 2 hidden layers of dimensions 800 and 500
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800,500]"
The network will have the dimensions [784,800,500,10]
784 is the input size of digit images (28pix x 28pix = 784)
10 is the number of digits
'''
net_dims = ast.literal_eval(sys.argv[1])
net_dims.append(10) # Adding the digits layer with dimensionality = 10
print("Network dimensions are:" + str(net_dims))
# getting the subset dataset from MNIST
Original_train_data, Original_train_label, test_data, test_label = \
mnist(noTrSamples=6000, noTsSamples=1000, \
digit_range=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], \
noTrPerClass=600, noTsPerClass=100)
train_data = Original_train_data[:, :500]
train_label = Original_train_label[:, :500]
validation_data = Original_train_data[:, 500:600]
validation_label = Original_train_label[:, 500:600]
for i in range(600, 6000, 600):
train_data = np.hstack((train_data, Original_train_data[:, i:i + 500]))
train_label = np.hstack((train_label, Original_train_label[:, i:i + 500]))
validation_data = np.hstack((validation_data, Original_train_data[:, i + 500:i + 600]))
validation_label = np.hstack((validation_label, Original_train_label[:, i + 500:i + 600]))
learning_rate = 0.5
num_iterations = 500
costs, parameters,valid_costs = multi_layer_network(train_data, train_label,validation_data,validation_label, net_dims, \
num_iterations=num_iterations, learning_rate=learning_rate)
# compute the accuracy for training set and testing set
train_Pred = classify(train_data, parameters)
test_Pred = classify(test_data, parameters)
val_Pred = classify(validation_data, parameters)
trAcc = (1 - np.count_nonzero(train_Pred - train_label) / float(train_Pred.shape[1])) * 100
teAcc = (1 - np.count_nonzero(test_Pred - test_label) / float(test_Pred.shape[1])) * 100
valAcc = (1 - np.count_nonzero(val_Pred - validation_label) / float(val_Pred.shape[1])) * 100
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
print("Accuracy for validation set is {0:0.3f} %".format(valAcc))
X = range(0, 500, 10)
plt.plot(X, costs, label="train data")
plt.plot(X,valid_costs,label="validation")
plt.xlabel("Iterations")
plt.ylabel("Cost")
plt.legend()
plt.show()
def main():
start_time = time.time()
train_data, train_label, test_data, test_label = mnist(
noTrSamples=60000,
noTsSamples=10000,
digit_range=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
noTrPerClass=6000,
noTsPerClass=1000)
# print(param)
n_rows = train_data.shape[0]
n_cols = train_data.shape[1]
test_rows = test_data.shape[0]
test_cols = test_label.shape[1]
mean = 0.0
stddev = 0.2
noise = np.random.normal(mean, stddev, (n_rows, n_cols))
trX_noisy = train_data + noise
noise = np.random.normal(mean, stddev, (test_rows, test_cols))
tsX_noisy = test_data + noise
# fig = plt.figure()
# plt.imshow(train_data[:, 999].reshape(28, -1))
# plt.title("Training_Sample")
# plt.show()
# fig.savefig("Training_Sample")
#
# fig = plt.figure()
# plt.imshow(trX_noisy[:, 999].reshape(28, -1))
# plt.title("Noisy_Training_Sample")
# plt.show()
# fig.savefig("Noisy_Training_Sample")
n_in, m = train_data.shape
n_fin, m = train_data.shape
n_h = 1000
net_dims = [n_in, n_h, n_fin]
# initialize learning rate and num_iterations
learning_rate = 1
num_iterations = 1000
costs, parameters = two_layer_network(trX_noisy,
train_data,
net_dims,
num_iterations=num_iterations,
learning_rate=learning_rate)
# compute the accuracy for training set and testing set
# train_Pred = classify(train_data, parameters)
fig = plt.figure()
plt.imshow(test_data[:, 9999].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Test_Sample")
plt.show()
fig.savefig("Test_Sample")
fig = plt.figure()
plt.imshow(tsX_noisy[:, 9999].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Noisy_Test_Sample")
plt.show()
fig.savefig("Noisy_Test_Sample")
fig = plt.figure()
test_Pred = denoise(tsX_noisy, parameters)
plt.imshow(test_Pred[:, 9999].reshape(28, -1), cmap=plt.cm.binary)
plt.title("Denoised_Test_Sample")
plt.show()
fig.savefig("Denoised_Test_Sample")
print("Total execution time: %s minutes!!" %
((time.time() - start_time) // 60))
import matplotlib.pyplot as plt
from sklearn import datasets, svm, metrics
from load_mnist import mnist
import numpy as np
digit_range = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
train_data, train_label, val_data, val_label, test_data, test_label = \
mnist(noTrSamples=50, noValSamples=0, noTsSamples=1000,\
digit_range=digit_range,\
noTrPerClass=5, noValPerClass=0, noTsPerClass=100)
train_data = np.transpose(train_data)
train_label = np.transpose(train_label)
train_label = np.squeeze(train_label, axis=1)
test_data = np.transpose(test_data)
test_label = np.transpose(test_label)
test_label = np.squeeze(test_label, axis=1)
classifier = svm.SVC(C=5, gamma=0.05)
classifier.fit(train_data, train_label)
predicted = classifier.predict(test_data)
accuracy = metrics.accuracy_score(test_label, predicted)
def main():
'''
Trains a multilayer network for MNIST digit classification (all 10 digits)
To create a network with 1 hidden layer of dimensions 800
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800]"
The network will have the dimensions [784,800,10]
784 is the input size of digit images (28pix x 28pix = 784)
10 is the number of digits
To create a network with 2 hidden layers of dimensions 800 and 500
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800,500]"
The network will have the dimensions [784,800,500,10]
784 is the input size of digit images (28pix x 28pix = 784)
10 is the number of digits
'''
net_dims = ast.literal_eval( sys.argv[1] )
net_dims.append(10) # Adding the digits layer with dimensionality = 10
print("Network dimensions are:" + str(net_dims))
# getting the subset dataset from MNIST
train_data, train_label, test_data, test_label = \
mnist(noTrSamples=60000,noTsSamples=10000,\
digit_range=[0,1,2,3,4,5,6,7,8,9],\
noTrPerClass=6000, noTsPerClass=1000)
train_data, validation_data,train_label, validation_label= train_test_split(train_data.T,train_label.T,test_size=10000,train_size=50000,random_state=42)
learning_rate = 0.2
num_iterations = 1
epochs=1000
parameters = initialize_multilayer_weights(net_dims)
costvalues, costvalues_valid, times = [], [], []
for j in range(0,epochs):
start = time.process_time()
for i in range(0,10):
print ("EPOCH------------->=",j)
print ("BATCH------------->=",i)
mini_train_data=train_data[i*5000 : (i*5000)+5000]
mini_train_label=train_label[i*5000 : (i*5000)+5000]
mini_train_data=mini_train_data.T
mini_train_label=mini_train_label.T
costs,Vcosts,parameters = multi_layer_network(parameters,mini_train_data,mini_train_label,validation_data.T,validation_label.T, net_dims, num_iterations=num_iterations, learning_rate=learning_rate)
stop = time.process_time()
timetaken = stop - start
times.append(timetaken)
costvalues.append(costs)
costvalues_valid.append(Vcosts)
print("time taken for this epoch --->= ", timetaken)
train_Pred = classify(train_data.T, parameters)
test_Pred = classify(test_data, parameters)
trAcc = calculateAccuracy(train_Pred.T,train_label.T)
teAcc = calculateAccuracy(test_Pred.T,test_label)
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
print("Memory Usage : ",int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)/(784*1024), "MB")
plt.plot(np.squeeze(costvalues));
plt.plot(np.squeeze(costvalues_valid));
plt.show()
plt.plot(np.squeeze(times));
plt.show()
def main():
'''
Trains a multilayer network for MNIST digit classification (all 10 digits)
To create a network with 1 hidden layer of dimensions 800
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800]"
The network will have the dimensions [784,800,10]
784 is the input size of digit images (28pix x 28pix = 784)
10 is the number of digits
To create a network with 2 hidden layers of dimensions 800 and 500
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800,500]"
The network will have the dimensions [784,800,500,10]
784 is the input size of digit images (28pix x 28pix = 784)
10 is the number of digits
'''
net_dims = ast.literal_eval(sys.argv[1])
net_dims.append(10) # Adding the digits layer with dimensionality = 10
print("Network dimensions are:" + str(net_dims))
# getting the subset dataset from MNIST
data, label, test_data, test_label = \
mnist(noTrSamples=6000, noTsSamples=1000,
digit_range=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
noTrPerClass=600, noTsPerClass=100)
for i in range(10):
if i == 0:
train_data = data[:, :500]
val_data = data[:, 500:600]
train_label = label[:, :500]
val_label = label[:, 500:600]
else:
train_data = np.concatenate(
[train_data, data[:, 600 * (i):(500 + (i) * 600)]], axis=1)
val_data = np.concatenate(
[val_data, data[:, (500 + (i) * 600):(500 + (i) * 600) + 100]],
axis=1)
train_label = np.concatenate(
[train_label, label[:, 600 * (i):(500 + (i) * 600)]], axis=1)
val_label = np.concatenate([
val_label, label[:, (500 + (i) * 600):(500 + (i) * 600) + 100]
],
axis=1)
# initialize learning rate and num_iterations
learning_rate = 1.5
num_iterations = 500
cost_tr, cost_val, parameters = multi_layer_network(
train_data,
train_label,
val_data,
val_label,
net_dims,
num_iterations=num_iterations,
learning_rate=learning_rate)
# compute the accuracy for training set and testing set
train_Pred = classify(train_data, parameters)
test_Pred = classify(test_data, parameters)
trAcc = accuracy(train_Pred, train_label)
teAcc = accuracy(test_Pred, test_label)
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
plt.xlabel("iterations")
plt.ylabel("cost")
plt.title("Multiclass - iterations vs cost for train and validation data")
plt.plot(cost_tr, "-g", label='train')
plt.plot(cost_val, ":b", label='val')
plt.legend()
plt.show()
# python 3
import matplotlib.pyplot as plt
import sys
from load_mnist import mnist
from sklearn import svm
import warnings
# Save all the Print Statements in a Log file.
print_out = sys.stdout
log_file = open("BaseLine_SVM_Results.txt", "a")
sys.stdout = log_file
train_data, train_label, test_data, test_label = mnist(
noTrSamples=60000,
noTsSamples=10000,
digit_range=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
noTrPerClass=6000,
noTsPerClass=1000)
# print(train_data.shape, test_data.shape)
# plt.figure()
# plt.imshow(train_data[:, 0].reshape(28, -1), cmap=plt.cm.binary)
# plt.colorbar()
# plt.grid(False)
# plt.show()
print('\nSVM Classifier with gamma = 0.1; Kernel = polynomial')
with warnings.catch_warnings():
warnings.simplefilter("ignore")
clf = svm.SVC(gamma=0.1, kernel='poly')
clf.fit(train_data.T, train_label.T)
def main():
'''
Trains a multilayer network for MNIST digit classification (all 10 digits)
To create a network with 1 hidden layer of dimensions 800
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800]"
The network will have the dimensions [784,800,10]
784 is the input size of digit images (28pix x 2 8pix = 784)
10 is the number of digits
To create a network with 2 hidden layers of dimensions 800 and 500
Run the progam as:
python deepMultiClassNetwork_starter.py "[784,800,500]"
The network will have the dimensions [784,800,500,10]
784 is the input size of digit images (28pix x 28pix = 784)
10 is the number of digits
'''
# getting the subset dataset from MNIST
data, label, test_data, test_label = \
mnist(noTrSamples=6000,noTsSamples=1000,\
digit_range=[0,1,2,3,4,5,6,7,8,9],\
noTrPerClass=600, noTsPerClass=100)
# parse data into training and validation sets
# training data
train_data = np.array([])
train_label = np.array([])
train_data = data[:, 0:500]
train_label1 = label[:, 0:500]
for i in range(600, 6000, 600):
# print i
temp = data[:, i:i + 500]
temp2 = label[:, i:i + 500]
# print temp.shape
train_data = np.column_stack((train_data, temp))
train_label1 = np.column_stack((train_label1, temp2))
# train data -> 784*5000
# reshape train_lable from 1*5000 to 10*5000
# since instead of train_label[sample]=digit it should be train_label[sample][digit] = 1/0
column = train_label1.shape[1]
mainList = []
for i in range(0, column):
l = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
temp = int(train_label1[0][i])
l[temp] = 1
mainList.append(l)
train_label = np.array(mainList)
train_label = train_label.T
# print "Train labels",train_label.shape
# validation data
validation_data = np.array([])
validation_label = np.array([])
validation_data = data[:, 500:600]
validation_label1 = label[:, 500:600]
for i in range(1100, 6000, 600):
# print i
temp = data[:, i:i + 100]
temp2 = label[:, i:i + 100]
# print temp.shape
validation_data = np.column_stack((validation_data, temp))
validation_label1 = np.column_stack((validation_label1, temp2))
# train data -> 784*5000
# reshape train_lable from 1*5000 to 10*5000
# since instead of train_label[sample]=digit it should be train_label[sample][digit] = 1/0
column = validation_label1.shape[1]
mainList = []
for i in range(0, column):
l = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
temp = int(validation_label1[0][i])
l[temp] = 1
mainList.append(l)
validation_label = np.array(mainList)
validation_label = validation_label.T
# validation data -> 784*1000
# confirm data
# a =[0,0,0,0,0,0,0,0,0,0]
# for i in validation_label[0]:
# a[int(i)]+=1
# print a
# initialize learning rate and num_iterations
# varying learning rates.
num_iterations = 1000
rates = [0.1, 0.2, 0.5, 1.0, 10]
net_dims = ast.literal_eval(sys.argv[1])
# print net_dims
net_dims.append(10) # Adding the digits layer with dimensionality = 10
print "Network dimensions are:" + str(net_dims)
ax1 = plt.subplot2grid(shape=(2, 6), loc=(0, 0), colspan=2)
ax2 = plt.subplot2grid((2, 6), (0, 2), colspan=2)
ax3 = plt.subplot2grid((2, 6), (0, 4), colspan=2)
ax4 = plt.subplot2grid((2, 6), (1, 1), colspan=2)
ax5 = plt.subplot2grid((2, 6), (1, 3), colspan=2)
l = [ax1, ax2, ax3, ax4, ax5]
j = 0
for learning_rate in rates:
print "Learning Rate: ", learning_rate
costs, vcosts,parameters = multi_layer_network(train_data, train_label,validation_data,validation_label, net_dims, \
num_iterations=num_iterations, learning_rate=learning_rate)
# compute the accuracy for training set and testing set
train_Pred = classify(train_data, parameters)
test_Pred = classify(test_data, parameters)
validation_Pred = classify(validation_data, parameters)
trAcc = calculateAccuracy(train_label1, train_Pred)
teAcc = calculateAccuracy(test_label, test_Pred)
vaAcc = calculateAccuracy(validation_label1, validation_Pred)
print "Accuracy for training set is {0:0.3f} %".format(trAcc)
print "Accuracy for Validation set is {0:0.3f} %".format(vaAcc)
print "Accuracy for testing set is {0:0.3f} %".format(teAcc)
### CODE HERE to plot costs
iterations = [i for i in range(0, 1001, 10)]
# ,iterations,vcosts,'b',label='Validation cost'
l[j].plot(iterations, costs, 'r', label='Training cost')
l[j].plot(iterations, vcosts, 'b', label='Validation cost')
title = "Learning rate: ", learning_rate
l[j].set_title(title)
l[j].set_xlabel('Iterations')
l[j].set_ylabel('Costs')
l[j].legend()
j += 1
plt.show()
def main():
start_time = time.time()
train_data, train_label, test_data, test_label = mnist()
# print(param)
n_rows = train_data.shape[0]
n_cols = train_data.shape[1]
test_rows = test_data.shape[0]
test_cols = test_label.shape[1]
mean = 0.0
stddev = 0.4
noise = np.random.normal(mean, stddev, (n_rows, n_cols))
trX_noisy = train_data + noise
noise = np.random.normal(mean, stddev, (test_rows, test_cols))
tsX_noisy = test_data + noise
# fig = plt.figure()
# plt.imshow(train_data[:, 999].reshape(28, -1))
# plt.title("Training_Sample")
# plt.show()
# fig.savefig("Training_Sample")
#
# fig = plt.figure()
# plt.imshow(trX_noisy[:, 999].reshape(28, -1))
# plt.title("Noisy_Training_Sample")
# plt.show()
# fig.savefig("Noisy_Training_Sample")
n_in, m = train_data.shape
n_fin, m = train_data.shape
n_h = 1000
net_dims = [n_in, n_h, n_fin]
# initialize learning rate and num_iterations
learning_rate = 1
num_iterations = 1000
costs, parameters = two_layer_network(trX_noisy, train_data, net_dims, num_iterations=num_iterations,
learning_rate=learning_rate)
# compute the accuracy for training set and testing set
# train_Pred = classify(train_data, parameters)
fig = plt.figure()
plt.imshow(test_data[:, 99].reshape(28, -1))
plt.title("Test_Sample")
plt.show()
fig.savefig("Test_Sample")
fig = plt.figure()
plt.imshow(tsX_noisy[:, 99].reshape(28, -1))
plt.title("Noisy_Test_Sample")
plt.show()
fig.savefig("Noisy_Test_Sample")
fig = plt.figure()
test_Pred = denoise(tsX_noisy, parameters)
print("error of test sample" ,(error(test_Pred,test_data)))
plt.imshow(test_Pred[:, 99].reshape(28, -1))
plt.title("Denoised_Test_Sample")
plt.show()
fig.savefig("Denoised_Test_Sample")
plt.plot(costs, label = 'Training cost')
l2a = rectify(conv2d(l1, w2))
l2 = max_pool_2d(l2a, (2, 2))
l2 = dropout(l2, p_drop_conv)
l3a = rectify(conv2d(l2, w3))
l3b = max_pool_2d(l3a, (2, 2))
l3 = T.flatten(l3b, outdim=2)
l3 = dropout(l3, p_drop_conv)
l4 = rectify(T.dot(l3, w4))
l4 = dropout(l4, p_drop_hidden)
pyx = softmax(T.dot(l4, w_o))
return l1, l2, l3, l4, pyx
trX, teX, trY, teY = mnist(onehot=True)
trX = trX.reshape(-1, 1, 28, 28)
teX = teX.reshape(-1, 1, 28, 28)
X = T.tensor4(dtype='float64')
Y = T.matrix()
w = init_weights((32, 1, 3, 3))
w2 = init_weights((64, 32, 3, 3))
w3 = init_weights((128, 64, 3, 3))
w4 = init_weights((128 * 3 * 3, 625))
w_o = init_weights((625, 10))
noise_l1, noise_l2, noise_l3, noise_l4, noise_py_x = model(X, w, w2, w3, w4, 0.2, 0.5)
l1, l2, l3, l4, py_x = model(X, w, w2, w3, w4, 0., 0.)
def main():
# getting the subset dataset from MNIST
# binary classification for digits 1 and 7
digit_range = [1, 7]
data, data_label, test_data, test_label = \
mnist(noTrSamples=2400,noTsSamples=1000,\
digit_range=digit_range,\
noTrPerClass=1200, noTsPerClass=500)
print(data_label)
validation_costs = []
test_accuracies = []
train_data = np.concatenate((data[:, :1000], data[:, 1200:2200]), axis=1)
val_data = np.concatenate((data[:, 1000:1200], data[:, 2200:2400]), axis=1)
train_label = np.concatenate(
(data_label[0][:1000], data_label[0][1200:2200]))
val_label = np.concatenate(
(data_label[0][1000:1200], data_label[0][2200:2400]))
#convert to binary labels
train_label[train_label == digit_range[0]] = 0
train_label[train_label == digit_range[1]] = 1
test_label[test_label == digit_range[0]] = 0
test_label[test_label == digit_range[1]] = 1
val_label[val_label == digit_range[0]] = 0
val_label[val_label == digit_range[1]] = 1
n_in, m = train_data.shape
n_fin = 1
n_h = [100, 200, 500]
count = 1
num_iterations = 1000
iteration = [k for k in range(0, num_iterations, 10)]
#fig = plt.figure(figsize=(15, 10))
fig = plt.figure()
for n, j in enumerate(n_h):
net_dims = [n_in, j, n_fin]
# initialize learning rate and num_iterations
learning_rate = 0.01
out = []
out_t = []
out_v = []
print("Number of neurons in the hidden layer:", j)
print()
print("Training set")
costs, parameters = two_layer_network(train_data, train_label, net_dims, \
num_iterations=num_iterations, learning_rate=learning_rate)
print(
"------------------------------------------------------------------------------"
)
print("Validation set")
costs_val,parameters_val = two_layer_network(val_data, val_label, net_dims, \
num_iterations=num_iterations, learning_rate=learning_rate)
validation_costs.append(costs_val[-1])
print(
"------------------------------------------------------------------------------"
)
# compute the accuracy for training set and testing set
train_Pred = classify(train_data, parameters)
val_Pred = classify(val_data, parameters)
test_Pred = classify(test_data, parameters)
trAcc = None
teAcc = None
output_test = (test_label == test_Pred)
teAcc = (np.sum(output_test == True) / test_label.shape[1]) * 100
test_accuracies.append(teAcc)
output = (train_label == train_Pred)
trAcc = (np.sum(output == True) / train_label.shape[0]) * 100
output_val = (val_label == val_Pred)
valAcc = (np.sum(output_val == True) / val_label.shape[0]) * 100
print("Accuracy:")
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for validation set is {0:0.3f} %".format(valAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
print(
"------------------------------------------------------------------------------"
)
# CODE HERE TO PLOT costs vs iterations
if j == 100:
sub1 = fig.add_subplot(1, 3, 1)
sub1.plot(iteration, costs, label="Train")
sub1.plot(iteration, costs_val, label="Validation")
sub1.legend()
sub1.set_title('100 Neurons in the hidden layer')
sub1.set_xlabel('Iterations')
sub1.set_ylabel('Cost')
if j == 200:
sub2 = fig.add_subplot(1, 3, 2)
sub2.plot(iteration, costs, label="Train")
sub2.plot(iteration, costs_val, label="Validation")
sub2.legend()
sub2.set_title('200 Neurons in the hidden layer')
sub2.set_xlabel('Iterations')
sub2.set_ylabel('Cost')
if j == 500:
sub3 = fig.add_subplot(1, 3, 3)
sub3.plot(iteration, costs, label="Train")
sub3.plot(iteration, costs_val, label="Validation")
sub3.legend()
sub3.set_title('500 Neurons in the hidden layer')
sub3.set_xlabel('Iterations')
sub3.set_ylabel('Cost')
plt.show()
idx = validation_costs.index(min(validation_costs))
print("Mimimum validation cost:", min(validation_costs), "observed with ",
n_h[idx], "hidden layers")
print(
"Test accuracy using the architecture with the best validation cost:",
test_accuracies[idx])
def main():
# getting the subset dataset from MNIST
net_dims = ast.literal_eval(sys.argv[1])
net_dims.append(1)
digit_range = [1, 7]
train_data, train_label, data, label = \
mnist(noTrSamples=2000, noTsSamples=1400, \
digit_range=digit_range, \
noTrPerClass=1000, noTsPerClass=700)
test_data = np.concatenate((data[:, :500], data[:, 700:1200]), axis=1)
val_data = np.concatenate((data[:, 500:700], data[:, 1200:1400]), axis=1)
test_label = np.concatenate((label[:, :500], label[:, 700:1200]), axis=1)
val_label = np.concatenate((label[:, 500:700], label[:, 1200:1400]), axis=1)
val_label[val_label == digit_range[0]] = 0
val_label[val_label == digit_range[1]] = 1
train_label[train_label == digit_range[0]] = 0
train_label[train_label == digit_range[1]] = 1
test_label[test_label == digit_range[0]] = 0
test_label[test_label == digit_range[1]] = 1
n_in, m = train_data.shape
n_fin = 1
n_h = 500
net_dims = [n_in, n_h, n_fin]
# initialize learning rate and num_iterations
learning_rate = 0.1
num_iterations = 1000
costs, parameters,costs1 = two_layer_network(train_data, train_label,val_data, val_label, net_dims, \
num_iterations=num_iterations, learning_rate=learning_rate)
# compute the accuracy for training set and testing set
train_Pred = classify(train_data, parameters)
test_Pred = classify(test_data, parameters)
val_Pred=classify(val_data,parameters)
m1 = train_data.shape[1]
m2 = test_data.shape[1]
m3 = val_data.shape[1]
trAcc = 100 - (np.sum(np.absolute(train_label - train_Pred)) / m1) * 100
teAcc = 100 - (np.sum(np.absolute(test_label - test_Pred)) / m2) * 100
valAcc = 100 - (np.sum(np.absolute(val_label - val_Pred)) / m3) * 100
print("Accuracy for training set is {0:0.3f} %".format(trAcc))
print("Accuracy for testing set is {0:0.3f} %".format(teAcc))
print("Accuracy for validation set is {0:0.3f} %".format(valAcc))
points = np.arange(0, 100)
plt.plot(points, costs, label='train')
plt.plot(points, costs1, label='validation')
plt.xlabel('iterations')
plt.ylabel('cost')
plt.title("Error vs iterations")
plt.legend()
plt.show()
plt.savefig("Error vs iterations")
