def test_fc_backprop(self):
for case in self.fc_cases:
weight = case['weight']
out_c, in_c = weight.shape
bias = case['bias']
x = case['x'].astype(np.float32)
out = case['out']
grad_output = case['grad_output']
grad_x = case['grad_x']
grad_w = case['grad_w']
grad_b = case['grad_b']
fc = FullyConnected(d_in=in_c, d_out=out_c)
fc.W = weight
fc.b = bias
test_out = fc(x)
dv_x, dv_W, dv_b = fc.backward(x, grad_output)
self.assertTrue(np.allclose(out, test_out, rtol=0.0001))
self.assertTrue(np.allclose(grad_x, dv_x, rtol=0.001))
self.assertTrue(np.allclose(grad_w, dv_W))
self.assertTrue(np.allclose(grad_b, dv_b))
class Net:
def __init__(self):
self.parameter = dict()
self.grad = dict()
self.gradInput = dict()
self.Fc1 = FullyConnected()
self.Fc2 = FullyConnected()
self.Fc3 = FullyConnected()
self.Relu1 = Relu()
self.Relu2 = Relu()
self.Relu3 = Relu()
self.softmaxloss = SoftMaxLoss()
self.optimizer = Optimizer()
def initWeigth(self):
# self.parameter['w1'] = np.zeros(shape=(28*28, 32), dtype=np.float32)
# self.parameter['w2'] = np.zeros(shape=(32, 10), dtype=np.float32)
# self.parameter['b1'] = np.zeros(shape=(32), dtype=np.float32)
# self.parameter['b2'] = np.zeros(shape=(10), dtype=np.float32)
self.parameter['w1'] = np.random.rand(28 * 28, 32) * np.sqrt(
1 / (28 * 28 + 32))
self.parameter['w2'] = np.random.rand(32, 16) * np.sqrt(1 / (32 + 16))
self.parameter['w3'] = np.random.rand(16, 10) * np.sqrt(1 / (16 + 10))
self.parameter['b1'] = np.random.rand(32)
self.parameter['b2'] = np.random.rand(16)
self.parameter['b3'] = np.random.rand(10)
# self.parameter['w1'] = np.random.uniform(low=0.5, high=15, size = (28 * 28, 32))
# self.parameter['w2'] = np.random.uniform(low=0.3, high=15, size = (32, 16))
# self.parameter['w3'] = np.random.uniform(low=0.1, high=2, size = (16, 10))
# self.parameter['b1'] = np.random.uniform(low=0, high=1, size = (32) )
# self.parameter['b2'] = np.random.uniform(low=0., high=1, size = (16) )
# self.parameter['b3'] = np.random.uniform(low=0., high=1, size = (10) )
def ForwardPass(self, image):
fc1, fca2 = self.Fc1.forward(
input=image,
weight=self.parameter["w1"],
bias=self.parameter["b1"]) # in = 1x1024, out = 1x32
relu1, _ = self.Relu1.forward(fc1)
fc2, _ = self.Fc2.forward(
input=relu1,
weight=self.parameter["w2"],
bias=self.parameter["b2"]) # in = 1x32 , out = 1x10
relu2, _ = self.Relu2.forward(fc2)
fc3, _ = self.Fc3.forward(
input=relu2,
weight=self.parameter["w3"],
bias=self.parameter["b3"]) # in = 1x32 , out = 1x10
relu3, _ = self.Relu3.forward(fc3)
out = Softmax.forward(relu3)
return out
def BackwordPass(self, prediction):
dout = self.softmaxloss.backward(prediction)
dout = self.Relu3.backward(dout)
delta3, dw3, db3 = self.Fc3.backward(dout=dout)
relu_delta2 = self.Relu2.backward(delta3)
delta2, dw2, db2 = self.Fc2.backward(dout=relu_delta2)
relu_delta1 = self.Relu1.backward(delta2)
delta1, dw1, db1 = self.Fc1.backward(relu_delta1)
self.grad['dw1'] = dw1
self.grad['dw2'] = dw2
self.grad['dw3'] = dw3
self.grad['db1'] = db1
self.grad['db2'] = db2
self.grad['db3'] = db3
def Train(self, image, label):
data = np.array(image, dtype=np.float32)
data = data.flatten()
input_data = np.reshape(data, (1, data.size))
label = np.reshape(label, (1, label.size))
output = self.ForwardPass(input_data)
# print(output)
loss = self.softmaxloss.forward(output, label)
# print(loss)
#print(loss)
self.BackwordPass(output)
return loss, output
def Test(self, testImages, testLabelsHotEncoding):
numberOfImages = testImages.shape[0]
# numberOfImages = 10
avgloss = 0
avgacc = 0
count = 0
for iter in range(numberOfImages):
image = testImages[iter, :, :]
labels = testLabelsHotEncoding[iter, :]
data = np.array(image, dtype=np.float32)
data = data.flatten()
input_data = np.reshape(data, (1, data.size))
labels = np.reshape(labels, (1, labels.size))
output = self.ForwardPass(input_data)
pred = np.argmax(output[0])
gt = np.argmax(labels[0])
loss = self.softmaxloss.forward(output, labels)
if pred == gt:
count += 1
#print("True")
avgloss += loss
avgacc = float(count) / float(numberOfImages)
avgloss = float(avgloss) / float(numberOfImages)
print("Test Accuracy: ", avgacc)
print("Test Loss: ", avgloss)
return avgloss, avgacc
def start_training_mnist(self,
data_folder,
batch_size,
learning_rate,
NumberOfEpoch,
display=1000):
self.initWeigth()
self.optimizer.initADAM(3, 3)
trainingImages, trainingLabels = Dataloader.loadMNIST(
'train', data_folder)
testImages, testLabels = Dataloader.loadMNIST('t10k', data_folder)
trainLabelsHotEncoding = Dataloader.toHotEncoding(trainingLabels)
testLabelsHotEncoding = Dataloader.toHotEncoding(testLabels)
numberOfImages = trainingImages.shape[0]
# numberOfImages = 10
pEpochTrainLoss = []
pEpochTrainAccuracy = []
pEpochTestLoss = []
pEpochTestAccuracy = []
print("Training started")
t = 0
for epoch in range(NumberOfEpoch):
avgLoss = 0
trainAcc = 0.0
count = 0.0
countacc = 0.0
pIterLoss = []
print("##############EPOCH : {}##################".format(epoch))
for iter in range(numberOfImages):
t += 1
image = trainingImages[iter, :, :]
labels = trainLabelsHotEncoding[iter, :]
loss, output = self.Train(image, labels)
self.parameter = self.optimizer.ADAM(self.parameter, self.grad,
learning_rate, t)
self.parameter = self.optimizer.l2_regularization(
self.parameter, 0.001)
output = output[0]
pred = np.argmax(output)
gt = np.argmax(labels)
if pred == gt:
count += 1.0
countacc += 1.0
#print("True")
# self.parameter = self.optimizer.SGD(self.parameter, self.grad, learning_rate)
pIterLoss.append(loss)
avgLoss += loss
if iter % display == 0:
print("Train Accuracy {} with prob : {}".format(
(countacc / float(display)), output[pred]))
print("Train Loss: ", loss)
countacc = 0.0
loss, acc = self.Test(testImages, testLabelsHotEncoding)
trainAcc = (float(count) / float(numberOfImages))
print("##################Overall Accuracy & Loss Calculation")
print("TrainAccuracy: ", trainAcc)
print("TrainLoss: ", (float(avgLoss) / float(numberOfImages)))
avgtestloss, avgtestacc = self.Test(testImages,
testLabelsHotEncoding)
totaloss = float(avgLoss) / float(numberOfImages)
pEpochTrainLoss.append(totaloss)
pEpochTrainAccuracy.append(trainAcc)
pEpochTestLoss.append(avgtestloss)
pEpochTestAccuracy.append(avgtestacc)
x_axis = np.linspace(0, epoch, len(pEpochTrainLoss), endpoint=True)
plt.semilogy(x_axis, pEpochTrainLoss)
plt.xlabel('epoch')
plt.ylabel('loss')
plt.draw()
file = open("Weightparameter1.pkl", "wb")
file.write(pickle.dumps(self.parameter))
file.close()
fill2 = open("parameter.pkl", "wb")
fill2.write(
pickle.dumps((pEpochTrainAccuracy, pEpochTrainLoss,
pEpochTestAccuracy, pEpochTestLoss)))
fill2.close()
def start_training(self, datafolder, batchsize, learning_rate,
number_of_epoch):
pass