def backward(self, error_tensor):
# error_tensor has the same shape of input_tensor
# for convolutinoal layer
if len(self.input_shape) is 4:
error_tensor = self.reformat(error_tensor)
self.gradient_weights = (error_tensor *
self.normed_input_tensor).sum(axis=0)
self.gradient_bias = error_tensor.sum(axis=0)
gradient_input = Helpers.compute_bn_gradients(error_tensor,
self.input_tensor,
self.weights,
self.preMean,
self.preVariance)
if len(self.input_shape) is 4:
# reverse
gradient_input = self.reformat(gradient_input)
# update
if self._optimizer is not None:
self.weights = self.weights_optimizer.calculate_update(
self.weights, self.gradient_weights)
self.bias = self.bias_optimizer.calculate_update(
self.bias, self.gradient_bias)
return gradient_input
def test_gradient(self):
input_tensor = np.abs(np.random.random(self.label_tensor.shape))
layers = list()
layers.append(Loss.CrossEntropyLoss())
difference = Helpers.gradient_check(layers, input_tensor,
self.label_tensor)
self.assertLessEqual(np.sum(difference), 1e-4)
def backward(self, error_tensor):
# convolutional: 4D -> 2D
original_tensor = copy.deepcopy(error_tensor)
if len(error_tensor.shape) == 4:
error_tensor = self.reformat(error_tensor)
# grad w.r.t. weights & bias
self.gradient_weights = np.sum(error_tensor * self.x_norm,
axis=0,
keepdims=True)
self.gradient_bias = np.sum(error_tensor, axis=0, keepdims=True)
if self.optimizer is not None:
self.weights = self.optimizer.calculate_update(
self.weights, self.gradient_weights)
self.bias = self.optimizer.calculate_update(
self.bias, self.gradient_bias)
# grad w.r.t. input
error = Helpers.compute_bn_gradients(error_tensor,
self.input,
self.weights,
self.mu,
self.var,
eps=np.finfo(float).eps)
# convolutional: 2D -> 4D
if len(original_tensor.shape) == 4:
error = self.reformat(error)
return error
def test_gradient(self):
input_tensor = np.abs(np.random.random((self.batch_size, self.input_size)))
input_tensor *= 2.
input_tensor -= 1.
layers = list()
layers.append(ReLU.ReLU())
layers.append(L2Loss())
difference = Helpers.gradient_check(layers, input_tensor, self.label_tensor)
self.assertLessEqual(np.sum(difference), 1e-5)
def test_gradient_weights(self):
input_tensor = np.abs(
np.random.random((self.batch_size, self.input_size)))
layers = list()
layers.append(
FullyConnected.FullyConnected(self.input_size, self.categories))
layers.append(L2Loss())
difference = Helpers.gradient_check_weights(layers, input_tensor,
self.label_tensor, False)
self.assertLessEqual(np.sum(difference), 1e-5)
def test_data_access(self):
net = NeuralNetwork.NeuralNetwork(Optimizers.Sgd(1))
categories = 3
input_size = 4
net.data_layer = Helpers.IrisData(50)
net.loss_layer = Loss.CrossEntropyLoss()
fcl_1 = FullyConnected.FullyConnected(input_size, categories)
net.append_layer(fcl_1)
net.append_layer(ReLU.ReLU())
fcl_2 = FullyConnected.FullyConnected(categories, categories)
net.append_layer(fcl_2)
net.append_layer(SoftMax.SoftMax())
out = net.forward()
out2 = net.forward()
self.assertNotEqual(out, out2)
def test_iris_data(self):
net = NeuralNetwork.NeuralNetwork(Optimizers.Sgd(1e-3))
categories = 3
input_size = 4
net.data_layer = Helpers.IrisData(50)
net.loss_layer = Loss.CrossEntropyLoss()
fcl_1 = FullyConnected.FullyConnected(input_size, categories)
net.append_layer(fcl_1)
net.append_layer(ReLU.ReLU())
fcl_2 = FullyConnected.FullyConnected(categories, categories)
net.append_layer(fcl_2)
net.append_layer(SoftMax.SoftMax())
net.train(4000)
plt.figure(
'Loss function for a Neural Net on the Iris dataset using SGD')
plt.plot(net.loss, '-x')
plt.show()
data, labels = net.data_layer.get_test_set()
results = net.test(data)
index_maximum = np.argmax(results, axis=1)
one_hot_vector = np.zeros_like(results)
for i in range(one_hot_vector.shape[0]):
one_hot_vector[i, index_maximum[i]] = 1
correct = 0.
wrong = 0.
for column_results, column_labels in zip(one_hot_vector, labels):
if column_results[column_labels > 0].all() > 0:
correct += 1
else:
wrong += 1
accuracy = correct / (correct + wrong)
print('\nOn the Iris dataset, we achieve an accuracy of: ' +
str(accuracy * 100) + '%')
self.assertGreater(accuracy, 0.8)
def backward(self, error_tensor):
if self.isconv:
error_tensor_reformatted = self.reformat(error_tensor)
else:
error_tensor_reformatted = np.copy(error_tensor)
self.grad_x = np.zeros_like(error_tensor)
grad_x_hat = error_tensor_reformatted * self.weights
grad_x_reformatted = np.zeros_like(error_tensor_reformatted)
self.gradient_weights = np.zeros_like(self.weights)
self.gradient_bias = np.zeros_like(self.bias)
grad_x_reformatted = Helpers.compute_bn_gradients(
error_tensor_reformatted,
self.input_reformatted,
self.weights,
self.mean,
self.var,
eps=self.eps)
self.gradient_weights = np.sum(error_tensor_reformatted * self.x_hat,
axis=0)
self.gradient_bias = np.sum(error_tensor_reformatted, axis=0)
# self.grad_var += -0.5 * self.grad_xhat[b] * (self.input_tensor[b] - self.mean) * np.power(
# (self.var + self.eps), -1.5)
# self.grad_mean += self.grad_xhat[b] * (-1.0 / np.sqrt(self.var + self.eps))
if self.optimizer is not None:
self.weights = self.optimizer.calculate_update(
self.weights, self.gradient_weights)
self.bias = self.optimizer.calculate_update(
self.bias, self.gradient_bias)
if self.isconv:
self.grad_x = self.reformat(grad_x_reformatted)
else:
self.grad_x = np.copy(grad_x_reformatted)
return np.copy(self.grad_x)
from Layers import Helpers
from Models.LeNet import build
import NeuralNetwork
import matplotlib.pyplot as plt
import os.path
batch_size = 50
mnist = Helpers.MNISTData(batch_size)
mnist.show_random_training_image()
if os.path.isfile(os.path.join('.', 'trained', 'LeNet.pk')): #地址必须添加"."和".pk"
net = NeuralNetwork.load(os.path.join('.', 'trained', 'LeNet.pk'), mnist)
else:
net = build()
net.data_layer = mnist
net.train(300)
NeuralNetwork.save(os.path.join('.', 'trained', 'LeNet.pk'), net)
plt.figure('Loss function for training LeNet on the MNIST dataset')
plt.plot(net.loss, '-x')
plt.show()
data, labels = net.data_layer.get_test_set()
results = net.test(data)
accuracy = Helpers.calculate_accuracy(results, labels)
print('\nOn the MNIST dataset, we achieve an accuracy of: ' +
str(accuracy * 100) + '%')