def build_model(self, input_shape, hidden_shape, output_shape, learning_rate, is_agent_mode_enabled):
self.input_layer = InputLayer()
self.hidden_layers = []
last_output_shape = input_shape
for shape in hidden_shape:
self.hidden_layers.append(HiddenLayer(shape, last_output_shape, sigmoid, learning_rate, is_agent_mode_enabled))
last_output_shape = shape
last_layer_output_shape = self.hidden_layers[-1].get_output_shape()
self.output_layer = OutputLayer(output_shape, last_layer_output_shape, sigmoid, learning_rate)
def __init__(self, input_size, output_size, layers=None):
self.input_layer = InputLayer(input_size)
self.current_layer = self.input_layer
if layers is not None:
for l in layers:
self.current_layer.add_next_layer(
FCLayer(self.current_layer, l, self.learning_rate))
self.current_layer = self.current_layer.get_next_layer()
self.current_layer.add_next_layer(
OutputLayer(self.current_layer, output_size, self.learning_rate))
self.current_layer = self.current_layer.get_next_layer()
self.output_layer = self.current_layer
def __init__(self, learningrate, n_features, num_of_hidden_layers, Ws, bs):
self.learning_rate = learningrate
self.n_class = Ws[-1].shape[1]
self.features = n_features
self.L = num_of_hidden_layers + 1
self.input_layer = InputLayer()
self.output_layer = OutputLayer()
self.hidden_layers = np.array([])
self.connections = np.array([])
for i in range(num_of_hidden_layers):
self.hidden_layers = np.append(self.hidden_layers, HiddenLayer())
for i in range(1, self.L + 1):
self.connections = np.append(self.connections, Connection())
self.connections[i - 1].set(Ws[i - 1], bs[i - 1], i - 1, i)
return
def buildNetwork(self):
"""
Builds the neural network with a fixed structure,
and a variable number of outputs.
"""
self.inputLayer = InputLayer()
convLayer = ConvolutionalLayer(5,10)
poolLayer = PoolingLayer(4)
reluLayer = ReluLayer()
convLayer2 = ConvolutionalLayer(4,20)
pool2Layer = PoolingLayer(2)
flattenLayer = FlattenLayer()
reluLayer2 = ReluLayer()
fullLayer = FullyConnectedLayer(20)
self.outputLayer = OutputLayer(self.numOutputs)
fullLayer.connect(self.outputLayer)
flattenLayer.connect(fullLayer)
reluLayer2.connect(flattenLayer)
pool2Layer.connect(reluLayer2)
convLayer2.connect(pool2Layer)
reluLayer.connect(convLayer2)
poolLayer.connect(reluLayer)
convLayer.connect(poolLayer)
self.inputLayer.connect(convLayer)
def _init_net(self):
input_data_num = self.net_structure[0] #输入层个数
input_layer = InputLayer(input_data_num) #初始化输入层
self.input_layer = input_layer
layer_num = np.shape(self.net_structure)[0]
last_layer = input_layer
#创建网路
for i in range(1, layer_num):
layer = Layer(self.param, self.net_structure[i], i)
layer.set_last_layer(last_layer)
last_layer = layer
output_layer = OutputLayer(self.param.diff_err_func,
self.param.err_func) #初始化输出层
output_layer.set_last_layer(last_layer)
self.output_layer = output_layer
def __init__(self,
list_layers,
activation_function,
learning_rate,
batch_size=32):
self.list_layers = list_layers
self.learning_rate = learning_rate
self.layers = []
self.batch_size = batch_size
self.layers.append(InputLayer(list_layers[0]))
for i in range(1, len(list_layers)):
self.layers.append(
NeuralLayer(list_layers[i],
list_layers[i - 1],
activation_function,
initial_weights='random'))
"""
class Network:
__A = (1, 0, 0, 0, 0, 0, 0, 0, 0, 0)
__B = (0, 1, 0, 0, 0, 0, 0, 0, 0, 0)
__C = (0, 0, 1, 0, 0, 0, 0, 0, 0, 0)
__D = (0, 0, 0, 1, 0, 0, 0, 0, 0, 0)
__E = (0, 0, 0, 0, 1, 0, 0, 0, 0, 0)
__F = (0, 0, 0, 0, 0, 1, 0, 0, 0, 0)
__G = (0, 0, 0, 0, 0, 0, 1, 0, 0, 0)
__H = (0, 0, 0, 0, 0, 0, 0, 1, 0, 0)
__I = (0, 0, 0, 0, 0, 0, 0, 0, 1, 0)
__J = (0, 0, 0, 0, 0, 0, 0, 0, 0, 1)
def __init__(self, training_set, test_set, option):
self.__inLayer = InputLayer(training_set, test_set, option)
self.__hidLayer = HiddenLayer(28 * 28, 30, option)
self.__outLayer = OutputLayer(30, 10, option)
self.option = option
def set_hid_weights(self, weights):
self.__hidLayer.set_weights(weights)
return self
def set_out_weights(self, weights):
self.__outLayer.set_weights(weights)
return self
def set_hid_bias(self, bias):
self.__hidLayer.set_bias(bias)
return self
def set_out_bias(self, bias):
self.__outLayer.set_bias(bias)
return self
def train(self, last_time):
loss = 0
for i in range(self.__inLayer.training_set_size()):
desired_output = self.get_desired_output(
self.__inLayer.get_training_label(i))
self.__outLayer.set_desired_output(desired_output)
inp = self.__inLayer.get_image(i)
if self.option.is_dropout():
prob = np.random.randint(0, 2, (1, 784))
inp = np.multiply(inp, prob)
self.__hidLayer.calc(inp)
hid = self.__hidLayer.get_output()
if self.option.is_dropout():
prob = np.random.randint(0, 2, (1, 30))
hid = np.multiply(hid, prob)
self.__outLayer.calc(hid)
loss += self.__outLayer.loss_function()
self.__outLayer.back_propagate(hid)
self.__hidLayer.back_propagate(inp, self.__outLayer)
if last_time:
np.savez_compressed('weights',
hid_weights=self.__hidLayer.get_weights(),
out_weights=self.__outLayer.get_weights(),
hid_bias=self.__hidLayer.get_bias(),
out_bias=self.__outLayer.get_bias())
return loss / self.__inLayer.training_set_size()
def test(self, last_time):
count = 0
loss = 0
for i in range(self.__inLayer.test_set_size()):
desired_output = self.get_desired_output(
self.__inLayer.get_test_label(i))
self.__outLayer.set_desired_output(desired_output)
inp = self.__inLayer.get_test_image(i)
self.__hidLayer.calc(inp)
self.__outLayer.calc(self.__hidLayer.get_output())
loss += self.__outLayer.loss_function()
must_be = self.__inLayer.get_test_label(i)
y_predict = np.zeros(10, dtype=np.int)
max_i = np.argmax(self.__outLayer.get_output())
y_predict[max_i] = 1
prediction = self.prediction(tuple(y_predict))
if must_be == prediction:
count += 1
if last_time:
print("accuracy: %.2f%%" %
(100 * count / self.__inLayer.test_set_size()))
return loss / self.__inLayer.test_set_size()
def get_desired_output(self, desired):
if desired == "A":
return Network.__A
elif desired == "B":
return Network.__B
elif desired == "C":
return Network.__C
elif desired == "D":
return Network.__D
elif desired == "E":
return Network.__E
elif desired == "F":
return Network.__F
elif desired == "G":
return Network.__G
elif desired == "H":
return Network.__H
elif desired == "I":
return Network.__I
elif desired == "J":
return Network.__J
def prediction(self, output):
if output == Network.__A:
return "A"
elif output == Network.__B:
return "B"
elif output == Network.__C:
return "C"
elif output == Network.__D:
return "D"
elif output == Network.__E:
return "E"
elif output == Network.__F:
return "F"
elif output == Network.__G:
return "G"
elif output == Network.__H:
return "H"
elif output == Network.__I:
return "I"
elif output == Network.__J:
return "J"
def __init__(self, training_set, test_set, option):
self.__inLayer = InputLayer(training_set, test_set, option)
self.__hidLayer = HiddenLayer(28 * 28, 30, option)
self.__outLayer = OutputLayer(30, 10, option)
self.option = option
class ConvolutionalNetwork:
def __init__(self,numClasses):
self.inputLayer = None
self.outputLayer = None
self.numOutputs = numClasses
self.dataLoader = CifarDataLoader()
self.error = 0
self.error_plotX = []
self.error_plotY = []
self.precisionX = []
self.precisionY = []
def buildNetwork(self):
"""
Builds the neural network with a fixed structure,
and a variable number of outputs.
"""
self.inputLayer = InputLayer()
convLayer = ConvolutionalLayer(5,10)
poolLayer = PoolingLayer(4)
reluLayer = ReluLayer()
convLayer2 = ConvolutionalLayer(4,20)
pool2Layer = PoolingLayer(2)
flattenLayer = FlattenLayer()
reluLayer2 = ReluLayer()
fullLayer = FullyConnectedLayer(20)
self.outputLayer = OutputLayer(self.numOutputs)
fullLayer.connect(self.outputLayer)
flattenLayer.connect(fullLayer)
reluLayer2.connect(flattenLayer)
pool2Layer.connect(reluLayer2)
convLayer2.connect(pool2Layer)
reluLayer.connect(convLayer2)
poolLayer.connect(reluLayer)
convLayer.connect(poolLayer)
self.inputLayer.connect(convLayer)
def guess(self,input):
"""
Receive an image and returns the guess of
the network to that images.
"""
self.inputLayer.forwardPropagation(input)
return self.outputLayer.getOutput()
def backPropagate(self,expected_output):
"""
Starts the propagation through the network.
"""
self.outputLayer.backPropagation(expected_output)
def train(self, numEpochs):
"""
Trains the net under de CIFAR-10 dataset, width a cerrtain
numbber of epochs.
"""
file = open('results.txt', 'w')
images, classes = self.dataLoader.load_training_data()
numImages = images.shape[0]
training_images, training_classes = self.dataLoader.load_test_data()
classes_names = self.dataLoader.load_class_names()
self.error_plotX = []
self.error_plotY = []
for i in range(numEpochs):
self.error = 0
print("Epoch {0}: training".format(i+1))
for index in range(2000):
if index % 100 == 0:
print(index)
if classes[index] != 0 and classes[index] != 3:
continue
expected_output = np.zeros(10)
output = self.guess(images[index,:,:,:])
expected_output[classes[index]] = 1
self.backPropagate(expected_output)
self.error += (np.power(LA.norm(np.subtract(expected_output, output)), 2)/numImages)
self.error_plotX.append(i)
self.error_plotY.append(self.error)
print("Epoch {0}: test".format(i+1))
ratio = self.test_data(training_images,training_classes,classes_names)
self.precisionX.append(i)
self.precisionY.append(ratio)
file.write("Epoch {0}: precision={1} error={2}".format(i,ratio,self.error))
file.close()
def test_data(self,images,classes ,classes_names):
"""
Test the precision of the network.
"""
numImages = images.shape[0]
asserts = 0
total = 0
for index in range(1000):
if classes[index] != 0 and classes[index] != 3:
continue
total +=1
output = self.guess(images[index,:,:,:])
if np.argmax(output) == classes[index]:
asserts += 1
print(asserts,total)
return float(float(asserts)/float(numImages))
def plot_results(self):
"""
Plots the precision and error of the network.
"""
plt.figure()
plt.title("Precision", fontsize=20)
plt.xlabel('epochs')
plt.ylabel('ratio')
plt.plot(self.precisionX, self.precisionY)
plt.figure()
plt.title("Error", fontsize=20)
plt.xlabel('epochs')
plt.ylabel('error')
plt.plot(self.error_plotX, self.error_plotY)
plt.show()
class FullyConnectedNeuralNetwork:
def __init__(self, input_shape, hidden_shape, output_shape, learning_rate, is_agent_mode_enabled = False):
"""
Init and builds a fully-connected neural network.
Arguments:
- input_shape: number of neurons in input layer.
- hidden_shape: array with number of neurons for each hidden layer, [int, int ...].
- output_shape: number of neurons in output layer.
- learning_rate: learning rate used for optimization.
- is_agent_mode_enabled: whether or not the neurons should be agents in the hidden layer(s).
"""
self.build_model(input_shape, hidden_shape, output_shape, learning_rate, is_agent_mode_enabled)
def build_model(self, input_shape, hidden_shape, output_shape, learning_rate, is_agent_mode_enabled):
self.input_layer = InputLayer()
self.hidden_layers = []
last_output_shape = input_shape
for shape in hidden_shape:
self.hidden_layers.append(HiddenLayer(shape, last_output_shape, sigmoid, learning_rate, is_agent_mode_enabled))
last_output_shape = shape
last_layer_output_shape = self.hidden_layers[-1].get_output_shape()
self.output_layer = OutputLayer(output_shape, last_layer_output_shape, sigmoid, learning_rate)
def train(self, inputs, outputs):
for i in tqdm(range(inputs.shape[0])):
input_sample = inputs[i].reshape([inputs[i].shape[0], 1])
expected_output = outputs[i].reshape([outputs[i].shape[0], 1])
forward_pass_output = self.execute_forward_propagation(input_sample)
error = mean_squared_error_derivative(forward_pass_output, expected_output)
self.execute_backpropagation(error)
def execute_forward_propagation(self, input_vector):
layer_output = self.input_layer.execute_forward_pass(input_vector)
for layer in self.hidden_layers:
layer_output = layer.execute_forward_pass(layer_output)
layer_output = self.output_layer.execute_forward_pass(layer_output)
return layer_output
def execute_backpropagation(self, error):
last_hidden_output = self.hidden_layers[-1].get_output()
next_layer_weights = self.output_layer.execute_backward_pass(error, last_hidden_output)
self.hidden_layers.reverse()
for layer in self.hidden_layers:
next_layer_weights, error = layer.execute_backward_pass(error, next_layer_weights)
self.hidden_layers.reverse()
def test(self, input_vector):
return self.execute_forward_propagation(input_vector)
def print_n_parameters(self):
n_parameters = self.output_layer.get_n_parameters()
for layer in self.hidden_layers:
n_parameters += layer.get_n_parameters()
print("{} parameters.".format(str(n_parameters)))
class NeuralNetwork:
epochs = 100000
learning_rate = 0.01
def __init__(self, input_size, output_size, layers=None):
self.input_layer = InputLayer(input_size)
self.current_layer = self.input_layer
if layers is not None:
for l in layers:
self.current_layer.add_next_layer(
FCLayer(self.current_layer, l, self.learning_rate))
self.current_layer = self.current_layer.get_next_layer()
self.current_layer.add_next_layer(
OutputLayer(self.current_layer, output_size, self.learning_rate))
self.current_layer = self.current_layer.get_next_layer()
self.output_layer = self.current_layer
def train(self, data, labels, plot=True):
costs = []
for i in range(self.epochs):
r = np.random.randint(len(data) - 1)
x = data[r]
target = labels[r]
# reshape to 2d array (used for mnist data)
if x.ndim == 1:
x = x.reshape(1, x.shape[0])
y = self.input_layer.forward(x)
cost = cross_entropy(y, target)
self.output_layer.backward(target)
if i % 100 == 0:
print("Step: " + str(i) + ", cost: " + str(cost))
costs.append(cost)
if plot:
plt.plot(costs)
plt.title("Cost")
plt.show()
def batch_train(self, data, labels, batch_size=1, plot=True):
costs = []
for i in range(self.epochs):
r = np.random.randint(len(data) - batch_size - 1)
xs = data[r:r + batch_size]
targets = labels[r:r + batch_size]
dc_dy = 0
cost = 0
for j in range(batch_size):
x = xs[j]
target = targets[j]
# reshape to 2d array (used for mnist data)
if x.ndim == 1:
x = x.reshape(1, x.shape[0])
y = self.input_layer.forward(x)
cost = cost + cross_entropy(y, target)
dc_dy = dc_dy + (y - target)
dc_dy = dc_dy / batch_size
cost = cost / batch_size
self.output_layer.backward_batch(dc_dy)
if i % 100 == 0:
print("Step: " + str(i) + ", cost: " + str(cost))
costs.append(cost)
if (plot):
plt.plot(cost)
plt.title("Cost")
plt.show()
def test(self, data, labels):
total_tests = data.shape[0]
successes = 0
failures = 0
for i in range(total_tests):
x = data[i]
y_ = np.argmax(labels[i])
y = self.predict(x)
if y == y_:
successes = successes + 1
else:
failures = failures + 1
print("Accuracy: " + str(successes / total_tests * 100) + "% for " +
str(total_tests) + " tests.")
def predict(self, x):
return np.argmax(self.input_layer.forward(x))
class Network(object):
"""description of class"""
def __init__(self, learningrate, n_features, num_of_hidden_layers, Ws, bs):
self.learning_rate = learningrate
self.n_class = Ws[-1].shape[1]
self.features = n_features
self.L = num_of_hidden_layers + 1
self.input_layer = InputLayer()
self.output_layer = OutputLayer()
self.hidden_layers = np.array([])
self.connections = np.array([])
for i in range(num_of_hidden_layers):
self.hidden_layers = np.append(self.hidden_layers, HiddenLayer())
for i in range(1, self.L + 1):
self.connections = np.append(self.connections, Connection())
self.connections[i - 1].set(Ws[i - 1], bs[i - 1], i - 1, i)
return
def print(self):
for i in self.connections:
i.print()
return
def feed_forward(self, X):
# set A of input_layer
self.input_layer.set_A(X)
# Z(1) = W(1).T * A(0) + b(1);
self.hidden_layers[0].set_Z(self.connections[0].Z(self.input_layer.A) +
self.connections[0].b)
# start at 0
# A(1) = activation_function (Z(1));
self.hidden_layers[0].Z_to_A()
# loop
for i in range(1, len(self.hidden_layers)):
# i = 1 ...... L-1; L = 3
# Z(i) = W(i).T * A(i-1) + b(i);
self.hidden_layers[i].set_Z(
self.connections[i].Z(self.hidden_layers[i - 1].A) +
self.connections[i].b)
# A(i) = activation_function (Z(i));
self.hidden_layers[i].Z_to_A()
# Z(L) = W(L) * A[L-1] + b(L);
self.output_layer.set_Z(self.connections[self.L - 1].Z(
self.hidden_layers[self.L - 2].A))
# A(L) = activation_function(Z(L));
self.output_layer.Z_to_A()
return self.output_layer.A
def train(self, X, Y):
self.feed_forward(X)
E_L = []
dJ_dW = []
dJ_db = []
# calculate E(L) = 1/N * (Yhat - Y);
E_L.append(1.0 / X.shape[0] * (self.output_layer.A - Y))
dJ_dW.append(np.dot(self.hidden_layers[self.L - 2].A, E_L[-1].T))
# pd(J)/pd(W_L);
dJ_db.append(np.sum(E_L[-1], axis=1, keepdims=True))
# pd(J).pd(b_L);
for i in range(self.L - 1, 1, -1):
# Calculate E(L-1) = W(L-1) dot E(L).
E_L.append(np.dot(self.connections[i].W, E_L[-1]))
dJ_dW.append(np.dot(self.hidden_layers[i - 2].A, E_L[-1].T))
# pd(J)/pd(W_L_1);
dJ_db.append(np.sum(E_L[-1], axis=1, keepdims=True))
# pd(J)/pd(b_L_1);
# Calculate E(L-2) = W(L-2) dot E(L-1).
E_L.append(np.dot(self.connections[self.L - 2].W, E_L[-1]))
dJ_dW.append(np.dot(self.input_layer.A, E_L[-1].T))
# pd(J)/pd(W_L_2);
dJ_db.append(np.sum(E_L[-1], axis=1, keepdims=True))
# pd(J)/pd(b_L_2);
# w += -eta * pd(j)/pd(w)
# b += -eta + pd(j)/pd(w)
for i in range(self.L):
self.connections[i].update(self.learning_rate, dJ_dW[-i - 1],
dJ_db[-i - 1])
return
def predict(self, X, Y):
pred = self.feed_forward(X)
Y_new = 1.0 * np.argmax(pred, axis=0)
return np.sum(Y_new == Y)
return
def train_loop(self, X, Y, n_loop_time, mode='b', mini_batch=10):
if mode == 's':
for i in range(n_loop_time):
indx = np.random.randint(0, X.shape[0])
self.train(X[indx], Y[indx])
elif mode == 'b':
for i in range(n_loop_time):
self.train(X, Y)
else:
for i in range(n_loop_time):
indx = np.random.randint(0, X.shape[0], size=(mini_batch, ))
self.train(X[indx], Y[indx])
return