def __init__(self, userid):
self.inputLayer = InputLayer()
self.hiddenLayers = []
self.outputLayer = OutputLayer()
self.userid = userid
self.outputs = []
def addLayer(self,
file_name=None,
neurons=None,
output=False,
trainable=True):
"""Add a layer to the network
"""
if len(self.layers) == 0:
input_size = self.input_size
else:
input_size = self.layers[-1].num_neurons
if file_name:
if output:
self.layers.append(
OutputLayer(weight_file=file_name, trainable=trainable))
else:
self.layers.append(
HiddenLayer(weight_file=file_name, trainable=trainable))
else:
if output:
self.layers.append(
OutputLayer(num_neurons=neurons,
inputs=input_size,
trainable=trainable))
else:
self.layers.append(
HiddenLayer(num_neurons=neurons,
inputs=input_size,
trainable=trainable))
def __init__(self, userid): self.inputLayer = InputLayer() self.hiddenLayers = [] self.outputLayer = OutputLayer() self.userid = userid self.outputs = []
def __init__(self, train_x, train_y, test_x, test_y):
self.train_x = train_x
self.train_y = train_y
self.test_x = test_x
self.test_y = test_y
self.preprocess()
self.input_layer = InputLayer(self.train_x.shape[1])
self.output_layer = OutputLayer(len(self.train_y), self.input_layer)
self.input_layer.prev_layer = self.output_layer
self.accuracy = 0
self.loss = 0
self.add_layer(LayerDense(self.train_x.shape[1], 64))
self.add_layer(LayerDense(64, 32))
self.add_layer(LayerDense(32, len(self.train_y)))
self.optimizer = OptimizerAdam(learning_rate=0.05, decay=5e-7)
def new_network(cost_func, activation_func, struct, w_range, b_range):
net = Network()
net.cost_func = cost_func
net.activation_func = activation_func
net.struct = struct
# stops
for input_size, output_size in zip(struct, struct[1:-1]):
layer = Layer.new_layer(activation_func, input_size, output_size,
w_range, b_range)
net.layers.append(layer)
output_layer = OutputLayer.new_output_layer(cost_func, activation_func,
struct[-2], struct[-1],
w_range, b_range)
net.layers.append(output_layer)
return net
varF, F, init = bn.getRandomParameters(N + I, K, isConstantConnectivity=False)
res = Reservoir(I, L, bn_directory + 'time_series_data_3.csv',
directory + 'test_2018-08-03.csv', N, varF, F, init)
print('reservoir initialized')
functionInputs = [f_utils.getInputTuple(window) for i in range(O)]
functionsToApproximate, functionVectors = [], []
for i in range(O):
function_vector = f_utils.convertIntToBinaryVector(i, 2**window)
func = f_utils.convertVectorToFunction(function_vector)
functionsToApproximate.append(func)
# Only for printing purposes
functionVectors.append(function_vector)
output = OutputLayer(res, O, functionsToApproximate, functionInputs, delay,
dataStreamLength)
output.train(trainingSize)
output.test(testSize)
print(f'N = {N}')
print(f'K = {K}')
print(f'I = {I}')
print(f'L = {output.reservoir.L}')
print(f'window = {window}')
print(f'delay = {delay}')
print(f'dataStreamLength = {dataStreamLength}')
print(f'trainingSize = {trainingSize}')
print(f'testSize = {testSize}')
print(f'seed = {seed}')
print(f'O = {O}')
print()
def __init__(self, n_input, n_output):
self.input_layer = InputLayer(n_input)
self.output_layer = OutputLayer(n_output, self.input_layer)
self.input_layer.prev_layer = self.output_layer
self.accuracy = 0
self.loss = 0
# Initialize reservoir
bn_directory = os.getcwd() + '/BN_realization/'
directory = os.getcwd() + '/'
varF, F, init = bn.getRandomParameters(N_list[i] + I,
K + (L / N_list[i]),
isConstantConnectivity=False)
res = Reservoir(I, L, bn_directory + 'time_series_data_3.csv',
directory + 'experiment1_2018-08-03.csv', N_list[i],
varF, F, init)
# Train and test output layer
output = OutputLayer(res,
O,
functionsToApproximate,
functionInputs,
delay,
dataStreamLength,
nonRecursiveArgs=[[(0, 2)]])
output.train(trainingSize)
output.test(testSize)
# add results to data
print('adding data')
data[i, j] = sum([output.successRates[k] for k in range(O)]) / O
time = int(time.clock() - start)
# Write metadata to file
f_utils.printParameters(N_list, K, I, L_list, window, delay, dataStreamLength,
trainingSize, testSize, O, 'recursiveParity', seed,
time)
self.inputs[:, np.newaxis]
assert self.weights.shape == gradient.shape
self.weights -= gradient*learning_rate
def calculate_delta(self):
"""
Calculate the delta values associated with neurons in this hidden layer.
equivalent to g'(a(j))*sum(delta(k)w(kj))
"""
next_deltas = self.next_layer.get_deltas()
next_weights = self.next_layer.get_weights()
next_errors = np.sum(np.tile(next_deltas, (len(next_weights), 1))*\
next_weights, axis=1)
return self.output_wrt_inputs()*next_errors
if __name__ == "__main__":
from output_layer import OutputLayer
from functions import *
hl = HiddenLayer(2, 2, sigmoid_vectorized, sigmoid_derv_vectorized, 0)
ol = OutputLayer(2, 2, sigmoid_vectorized, sigmoid_derv_vectorized, error_derv, 0)
hl.set_next_layer(ol)
inputs = np.array([-0.7, 0.7])
targets = np.array([0.0, 0.5])
for i in range(10000):
print(hl.bias)
hl.forward_pass(inputs)
ol.forward_pass(hl.outputs)
ol.update_weight(targets, 0.5)
hl.update_weight(0.4)
print(ol.outputs)
class ANN:
def __init__(self, userid):
self.inputLayer = InputLayer()
self.hiddenLayers = []
self.outputLayer = OutputLayer()
self.userid = userid
self.outputs = []
def init(self, layers_data):
self.layers_data = layers_data
for i in range(1, len(layers_data) - 1):
self.hiddenLayers.append(HiddenLayer())
self.hiddenLayers[-1].init(layers_data[i - 1] + 1, layers_data[i])
self.outputLayer.init(layers_data[-2] + 1, layers_data[-1])
"""self.hiddenLayers[0].neurons[0].weights = [0.5, 0.4, -0.8]
self.hiddenLayers[0].neurons[1].weights = [0.9, 1.0, 0.1]
self.outputLayer.neurons[0].weights = [-1.2, 1.1, -0.3]"""
self.outputs = [0 for i in range(layers_data[-1])]
def activate(self, input_data):
self.inputLayer.init(input_data)
self.hiddenLayers[0].activate(self.inputLayer.data)
for i in range(1, len(self.hiddenLayers)):
self.hiddenLayers[i].activate(self.hiddenLayers[i - 1].outputs)
self.outputLayer.activate(self.hiddenLayers[-1].outputs)
self.outputs = self.outputLayer.outputs
def updateErrorGradients(self, desiredOutputs):
sum_errors = self.outputLayer.updateErrorGradients(desiredOutputs)
self.hiddenLayers[-1].updateErrorGradients(self.outputLayer)
for i in range(len(self.hiddenLayers) - 2, -1, -1):
self.hiddenLayers[i].updateErrorGradients(self.hiddenLayers[i + 1])
return sum_errors
def updateWeights_slope(self):
self.hiddenLayers[0].updateWeights_slope(self.inputLayer.data)
for i in range(1, len(self.hiddenLayers)):
self.hiddenLayers[i].updateWeights_slope(
self.hiddenLayers[i - 1].outputs)
self.outputLayer.updateWeights_slope(self.hiddenLayers[-1].outputs)
def updateWeights(self):
for i in range(len(self.hiddenLayers)):
self.hiddenLayers[i].updateWeights()
self.outputLayer.updateWeights()
def saveData(self):
data = []
data.append(self.layers_data)
for h in self.hiddenLayers:
for n in h.neurons:
data.append(n.weights)
for n in self.outputLayer.neurons:
data.append(n.weights)
if not models.Ann_net_data.objects.filter(userid=self.userid):
models.Ann_net_data.objects.create(userid=self.userid,
data=str(data))
else:
models.Ann_net_data.objects.filter(userid=self.userid).update(
data=str(data))
del data
return
### save neurons data
data = []
for h in self.hiddenLayers:
for n in h.neurons:
data.append(n.weights_slope_prev)
data.append(n.deltha)
for n in self.outputLayer.neurons:
data.append(n.weights_slope_prev)
data.append(n.deltha)
if not models.Ann_trainer_rp_data.objects.filter(userid=self.userid):
models.Ann_trainer_rp_data.objects.create(userid=self.userid,
nrndata=str(data))
else:
models.Ann_trainer_rp_data.objects.filter(
userid=self.userid).update(nrndata=str(data))
del data
def loadData(self):
return False
if not models.Ann_net_data.objects.filter(userid=self.userid):
return False
if not models.Ann_trainer_rp_data.objects.filter(userid=self.userid):
return False
### load net data
data = eval(models.Ann_net_data.objects.get(userid=self.userid).data)
self.init(data[0])
index = 1
for h in self.hiddenLayers:
for n in h.neurons:
n.weights = data[index]
index += 1
for n in self.outputLayer.neurons:
n.weights = data[index]
index += 1
del data
### load neurons data
data = eval(
models.Ann_trainer_rp_data.objects.get(userid=self.userid).nrndata)
index = 0
for h in self.hiddenLayers:
for n in h.neurons:
n.weights_slope_prev = data[index]
n.deltha = data[index + 1]
index += 2
for n in self.outputLayer.neurons:
n.weights_slope_prev = data[index]
n.deltha = data[index + 1]
index += 2
del data
return True
def __str__(self):
result = ""
result += "----------------->input layer :\n" + str(
self.inputLayer) + '\n'
for h in self.hiddenLayers:
result += "----------------->hidden layer :\n" + str(h) + '\n'
result += "----------------->output layer :\n" + str(
self.outputLayer) + '\n'
result += "----------------->outputs :\n" + str(self.outputs)
return result
weight_relation_vector = []
for _ in range(quantity_cluster):
t = [
round((random.uniform(0.4, 0.6)), 5) for _ in range(QUANTITY_INPUT)
]
weight_relation_vector.append(t)
return weight_relation_vector
if __name__ == '__main__':
epoch = 10
size_sample = 100
quantity_cluster = 5
output_layer = OutputLayer(form_weight_relation(quantity_cluster),
quantity_cluster)
input_layer = InputLayer(form_sample(size_sample),
output_layer,
size_sample=size_sample)
help_epoch = []
help_function = []
speed_function_start = 0.5
delta_function_start = 0.5
for i in range(epoch):
delta_function = (-i / (epoch + 1) + 1)
#delta_function = 1
speed_function = speed_function_start * e**(i / epoch)
k = input_layer.epoch(output_layer.weight_relation_vector,
output_layer.quantity_cluster, delta_function,
def add_output_layer(self, num_units):
l = OutputLayer(self.layers[-1], num_units)
self.layers.append(l)
class ANN: def __init__(self, userid): self.inputLayer = InputLayer() self.hiddenLayers = [] self.outputLayer = OutputLayer() self.userid = userid self.outputs = [] def init(self, layers_data): self.layers_data = layers_data for i in range(1, len(layers_data) - 1): self.hiddenLayers.append(HiddenLayer()) self.hiddenLayers[-1].init(layers_data[i - 1] + 1, layers_data[i]) self.outputLayer.init(layers_data[-2] + 1, layers_data[-1]) """self.hiddenLayers[0].neurons[0].weights = [0.5, 0.4, -0.8] self.hiddenLayers[0].neurons[1].weights = [0.9, 1.0, 0.1] self.outputLayer.neurons[0].weights = [-1.2, 1.1, -0.3]""" self.outputs = [0 for i in range(layers_data[-1])] def activate(self, input_data): self.inputLayer.init(input_data) self.hiddenLayers[0].activate(self.inputLayer.data) for i in range(1, len(self.hiddenLayers)): self.hiddenLayers[i].activate(self.hiddenLayers[i - 1].outputs) self.outputLayer.activate(self.hiddenLayers[-1].outputs) self.outputs = self.outputLayer.outputs def updateErrorGradients(self, desiredOutputs): sum_errors = self.outputLayer.updateErrorGradients(desiredOutputs) self.hiddenLayers[-1].updateErrorGradients(self.outputLayer) for i in range(len(self.hiddenLayers) - 2, -1, -1): self.hiddenLayers[i].updateErrorGradients(self.hiddenLayers[i + 1]) return sum_errors def updateWeights(self, learningRate): self.hiddenLayers[0].updateWeights(self.inputLayer.data, learningRate) for i in range(1, len(self.hiddenLayers)): self.hiddenLayers[i].updateWeights(self.hiddenLayers[i - 1].outputs, learningRate) self.outputLayer.updateWeights(self.hiddenLayers[-1].outputs, learningRate) def saveData(self): data = [] data.append(self.layers_data) for h in self.hiddenLayers: for n in h.neurons: data.append(n.weights) for n in self.outputLayer.neurons: data.append(n.weights) if not models.Ann_net_data.objects.filter(userid=self.userid): models.Ann_net_data.objects.create(userid=self.userid, data=str(data)) else: models.Ann_net_data.objects.filter(userid=self.userid).update(data=str(data)) del data def loadData(self): if not models.Ann_net_data.objects.filter(userid=self.userid): return data = eval(models.Ann_net_data.objects.get(userid=self.userid).data) self.init(data[0]) index = 1 for h in self.hiddenLayers: for n in h.neurons: n.weights = data[index] index += 1 for n in self.outputLayer.neurons: n.weights = data[index] index += 1 del data return True def __str__(self): result = "" result += "----------------->input layer :\n" + str(self.inputLayer) + '\n' for h in self.hiddenLayers: result += "----------------->hidden layer :\n" + str(h) + '\n' result += "----------------->output layer :\n" + str(self.outputLayer) + '\n' result += "----------------->outputs :\n" + str(self.outputs) return result
np.random.seed(3148)
# np.random.seed(45)
# n_nodes = 30
in_dim = XI.shape[1]
# --------------------------------------------------------------------------
# np.random.seed(3148) good combo
# n_nodes = 30
# L1 = layer('tanh',n_nodes ,in_dim, True) # no bias
# L2 = output_layer('sse','tanh',1, n_nodes, True) # bias
n_nodes = 60
L1 = Layer('tanh', n_nodes, in_dim)
L2 = OutputLayer('sse', 'linear', 1, n_nodes) #
eta = 0.001
iters, sErr = 200, np.Infinity
TrEnt = []
TeEnt = []
for i in range(iters):
# np.random.shuffle(X)
for obs in range(Xtr.shape[0]):
o0, y = Xtr[obs, :], float(Ytr[obs])
o1 = L1.feed(o0)
o2 = float(L2.feed(o1))
dCost_do2 = L2.costFuncDeriv(y, o2)
class ANN: def __init__(self, userid): self.inputLayer = InputLayer() self.hiddenLayers = [] self.outputLayer = OutputLayer() self.userid = userid self.outputs = [] def init(self, layers_data): for i in range(1, len(layers_data) - 1): self.hiddenLayers.append(HiddenLayer()) self.hiddenLayers[-1].init(layers_data[i - 1] + 1, layers_data[i]) self.outputLayer.init(layers_data[-2] + 1, layers_data[-1]) """self.hiddenLayers[0].neurons[0].weights = [0.5, 0.4, -0.8] self.hiddenLayers[0].neurons[1].weights = [0.9, 1.0, 0.1] self.outputLayer.neurons[0].weights = [-1.2, 1.1, -0.3]""" self.outputs = [0 for i in range(layers_data[-1])] def activate(self, input_data): self.inputLayer.init(input_data) self.hiddenLayers[0].activate(self.inputLayer.data) for i in range(1, len(self.hiddenLayers)): self.hiddenLayers[i].activate(self.hiddenLayers[i - 1].outputs) self.outputLayer.activate(self.hiddenLayers[-1].outputs) self.outputs = self.outputLayer.outputs def loadData(self): if not models.Ann_net_data.objects.filter(userid=self.userid): return data = eval(models.Ann_net_data.objects.get(userid=self.userid).data) self.init(data[0]) index = 1 for h in self.hiddenLayers: for n in h.neurons: n.weights = data[index] index += 1 for n in self.outputLayer.neurons: n.weights = data[index] index += 1 del data def __str__(self): result = "" result += "----------------->input layer :\n" + str(self.inputLayer) + '\n' for h in self.hiddenLayers: result += "----------------->hidden layer :\n" + str(h) + '\n' result += "----------------->output layer :\n" + str(self.outputLayer) + '\n' result += "----------------->outputs :\n" + str(self.outputs) return result
from neural_network import NeuralNetwork
from input_layer import InputLayer
from output_layer import OutputLayer
from hidden_layer import HiddenLayer
# test train xor function
model = NeuralNetwork(InputLayer(2), HiddenLayer(2, "sigmoid"),
OutputLayer(1, "sigmoid"))
train_input = [[1, 1], [1, 0], [0, 1], [0, 0]]
train_output = [0, 1, 1, 0]
model.train(train_input, train_output, 1, 0.1, 20)
model.predict([1, 1])
model.predict([1, 0])
model.predict([0, 1])
model.predict([0, 0])
