def setUp(self):
"""
Set up a small sequential network
"""
self.layer1 = FullLayer(5, 10)
self.relu1 = ReluLayer()
self.layer2 = FullLayer(10, 2)
self.softmax = SoftMaxLayer()
self.loss = CrossEntropyLayer()
self.model = Sequential(
(self.layer1, self.relu1, self.layer2, self.softmax), self.loss)
class TestMultiLayer(unittest.TestCase):
"""
test a multilayer network (check gradients)
"""
def setUp(self):
"""
Set up a small sequential network
"""
self.layer1 = FullLayer(5, 10)
self.relu1 = ReluLayer()
self.layer2 = FullLayer(10, 2)
self.softmax = SoftMaxLayer()
self.loss = CrossEntropyLayer()
self.model = Sequential(
(self.layer1,
self.relu1,
self.layer2,
self.softmax),
self.loss
)
def test_forward(self):
"""
Test forward pass with some fake input
"""
# make some fake input
x = np.array([[0.5, 0.2, 0.1, 0.3, 0.7],
[0.3, 0.1, 0.05, 0.8, 0.9]])
y = np.array([[0, 1],
[1, 0]])
# test layers individually
y1 = self.layer1.forward(x)
relu = self.relu1.forward(y1)
y2 = self.layer2.forward(relu)
soft = self.softmax.forward(y2)
loss = self.loss.forward(soft, y)
# test sequential model
y_model = self.model.forward(x, y)
self.assertAlmostEquals(loss, y_model)
def test_backward(self):
"""
Test the backward function using the numerical gradient
"""
# make some fake input
x = np.array([[0.5, 0.2, 0.1, 0.3, 0.7],
[0.3, 0.1, 0.05, 0.8, 0.9],
])
y = np.array([[0, 1],
[1, 0]])
out = self.model.forward(x, y)
grads = self.model.backward()
# test some gradients at the input
h = 0.001
for i in range(x.shape[0]):
for j in range(x.shape[1]):
new_x = np.copy(x)
new_x[i, j] += h
out2 = self.model.forward(new_x, y)
diff = (out2 - out) / h
print "######"
print diff
print grads[i, j]
print "######"
self.assertTrue(np.abs(diff - grads[i,j]) < 0.001)
relu1 = ReluLayer()
maxpool1= MaxPoolLayer()
layer2 = ConvLayer(16, 32, 3)
relu2 = ReluLayer()
maxpool2 = MaxPoolLayer()
loss1 = CrossEntropyLayer()
flatten = FlattenLayer()
layer3 = FullLayer(2048, 3)
softmax1 = SoftMaxLayer()
model = Sequential(
(
layer1,
relu1,
maxpool1,
layer2,
relu2,
maxpool2,
flatten,
layer3,
softmax1
),
loss1)
loss_epoch = []
accuracy_epoch =[]
# training and predicting
learning_rate = 0.1
loss_epoch = (model.fit(x_train, y_train, epochs=15, lr=learning_rate))
y_pred = model.predict(x_test)
from layers import (FullLayer, ReluLayer, SoftMaxLayer, CrossEntropyLayer)
from layers.sequential import Sequential
from layers.dataset import cifar100
(x_train, y_train), (x_test, y_test) = cifar100(1213268041)
test_accuracy=[]
epochs=20
lr=[0.1, 0.01, 0.2]
for i in lr:
model = Sequential(layers=(FullLayer(32 * 32 * 3, 2500),
ReluLayer(),
FullLayer(2500, 2000),
ReluLayer(),
FullLayer(2000, 1500),
ReluLayer(),
FullLayer(1500, 1000),
ReluLayer(),
FullLayer(1000, 500),
ReluLayer(),
FullLayer(500, 4),
SoftMaxLayer()),
loss=CrossEntropyLayer())
finalloss = model.fit(x_train, y_train,epochs=20, lr=i)
y_pred = model.predict(x_test)
accuracy=(np.mean(y_test == y_pred))
print('Accuracy: %.2f' % accuracy)
np.append(test_accuracy, accuracy)
plt.plot(range(epochs), finalloss, label=('Learning rate=',i))
plt.xlabel('epochs')
plt.ylabel('Training loss')
plt.title('Training loss Vs Epochs for the CIFAR100 data set')
layer6 = FullLayer(32*8*8, 4)
softmax = SoftMaxLayer()
loss = CrossEntropyLayer()
lr = 0.1
scores = []
n_epochs = 5
batch_size = 128
x_train = np.array(x_train)
y_train = np.array(y_train)
x_test = np.array(x_test)
y_test = np.array(y_test)
model = Sequential((layer1, relu1, layer2, layer3, relu2, layer4, layer5, layer6, softmax), loss)
loss2,loss3 = model.fit(x_train, y_train, x_test, y_test, n_epochs, lr, batch_size)
print("Train Loss", loss2)
print("Test Loss", loss3)
y_pred = model.predict(x_test)
print('Predictions', y_pred)
# y_train1= y_train.argmax(1)
print('Actual', np.argmax(y_test, axis=1))
scores.append(np.mean(y_pred == np.argmax(y_test, axis=1)))
print("Neural accuracy for n_epochs=" + str(n_epochs) + " and Learning Rate=" + str(lr) + " is: " + str(np.argmax(y_test, axis=1)))
scores = np.asarray(scores)
batch_size = 32
counter = 0
errork = np.zeros(k)
loss = np.zeros(shape=(k, epochs))
for train_index, test_index in kf.split(myX[np.arange(533), :, :, :]):
train_x, test_x = myX[train_index, :, :, :], myX[test_index, :, :, :]
train_y, test_y = y[train_index], y[test_index]
#training
print('Creating model with lr = ' + str(lr))
myNet = Sequential(
layers=(
ConvLayer(n_i=3, n_o=16, h=3),
ReluLayer(),
MaxPoolLayer(size=2),
ConvLayer(n_i=16, n_o=32, h=3),
ReluLayer(),
MaxPoolLayer(size=2),
FlattenLayer(),
FullLayer(n_i=12 * 12 * 32, n_o=6), # no neutral class:/
SoftMaxLayer()),
loss=CrossEntropyLayer())
print("Initiating training")
loss[counter, :] = myNet.fit(x=train_x,
y=train_y,
epochs=epochs,
lr=lr,
batch_size=batch_size)
myNet.save()
pred = myNet.predict(test_x)
accuracy = np.mean(pred == test_y)
#matplotlib.use('Agg')
import matplotlib.pyplot as plt
from layers.dataset import fer2013
#Import and process the input
(train_x, train_y), (val_x,val_y), (test_x, test_y) = fer2013()
lr = 0.1
epochs = 100
batch_size = 128
myNet = Sequential(layers=(ConvLayer(n_i=1,n_o=16,h=3),
ReluLayer(),
MaxPoolLayer(size=2),
ConvLayer(n_i=16,n_o=32,h=3),
ReluLayer(),
MaxPoolLayer(size=2),
FlattenLayer(),
FullLayer(n_i=12*12*32,n_o=7),
SoftMaxLayer()),
loss=CrossEntropyLayer())
myNet.load()
"""
pred = myNet.predict(val_x)
accuracy = np.mean(pred == val_y)
print('At learning rate = '+str(lr))
print('Validation Accuracy of Convolutional Neural Network = '+str(accuracy))
"""
forw = myNet.forward(test_x)
pred = myNet.predict(test_x)
from layers.cross_entropy import CrossEntropyLayer
from layers.dataset import fer2013
import numpy as np
(train_x, train_y), (val_x,val_y), (test_x, test_y) = fer2013()
#Since network is extremely slow, training on the first 1500 images
train_x = train_x[0:1500,:,:,:]
lr = 0.1
epochs = 100
batch_size = 128
myNet = Sequential(layers=(ConvLayer(n_i=1,n_o=16,h=3),
ReluLayer(),
MaxPoolLayer(size=2),
ConvLayer(n_i=16,n_o=32,h=3),
ReluLayer(),
MaxPoolLayer(size=2),
FlattenLayer(),
FullLayer(n_i=12*12*32,n_o=7),
SoftMaxLayer()),
loss=CrossEntropyLayer())
print("Initiating training")
loss = myNet.fit(x=train_x,y=train_y,epochs=epochs,lr=lr,batch_size=batch_size)
myNet.save()
pred = myNet.predict(val_x)
accuracy = np.mean(pred == val_y)
print('At learning rate = '+str(lr))
print('Accuracy of Convolutional Neural Network = '+str(accuracy))
from layers.full import FullLayer
from layers.softmax import SoftMaxLayer
from layers.cross_entropy import CrossEntropyLayer
from layers.sequential import Sequential
from layers.relu import ReluLayer
from layers.dataset import cifar100
from sklearn.metrics import accuracy_score
import numpy as np
(x_train, y_train), (x_test, y_test) = cifar100(1213391684)
layer1 = FullLayer(3072, 2052)
relu1 = ReluLayer()
layer2 = FullLayer(2052, 680)
relu2 = ReluLayer()
layer3 = FullLayer(680, 4)
softmax = SoftMaxLayer()
loss = CrossEntropyLayer()
model = Sequential((layer1, relu1, layer2, relu2, layer3, softmax), loss)
model.fit(x_train, y_train, epochs=25, lr=0.06)
out = model.predict(x_test)
y = np.reshape(y_test, (x_test.shape[0], 1))
print("y_test.shape:", y.shape)
print("out.shape:", out.shape)
t = accuracy_score(y, out)
print(y_test, "Predicted Val", out)
print('accuracy score', t)
score = (np.mean(out == y))
print("score", score)