class Network():
# INITIALIZE NEW NETWORK
def __init__(self,
damp=0.0001,
default_iter_num=10,
network_size=2,
layer_size=[18, 2, 11]):
np.set_printoptions(precision=4)
self.debug = Debug()
self.debug.off()
self.DAMP = damp
self.DEFAULT_ITER_NUM = default_iter_num
self.LAYER_SIZE = layer_size
self.NETWORK_SIZE = network_size
self.L = network_size - 1
self.nodes = [np.zeros(self.LAYER_SIZE[i]) for i in range(self.L + 1)]
self.weights = [
0.01 *
(np.random.rand(self.LAYER_SIZE[i + 1], self.LAYER_SIZE[i]) - 0.5)
for i in range(self.L)
]
self.bias = [
0.01 * (np.random.rand(self.LAYER_SIZE[i + 1]) - 0.5)
for i in range(self.L)
]
self.network_train_counter = 0
def save(self, filename):
data = {
"weights": array_to_list(self.weights),
"bias": array_to_list(self.bias)
}
with open(filename, 'w') as file:
json.dump(data, file, indent=4)
def load(self, filename):
data = {}
with open(filename, 'r') as file:
data = json.load(file)
w = data["weights"]
b = data["bias"]
self.weights = [np.array(w[i]) for i in range(len(w))]
self.bais = [np.array(b[i]) for i in range(len(b))]
def run(self, sample):
if (len(sample) != len(self.nodes[0])):
print(
"ERROR - sample length does not match first layer size.\nSample: "
+ str(sample))
self.nodes[0] = activation_func(sample)
#self.nodes[0] = sample
z = list(self.nodes)
for i in range(1, self.L + 1):
z[i] = np.dot(self.weights[i - 1],
self.nodes[i - 1]) + self.bias[i - 1]
self.nodes[i] = activation_func(z[i])
return self.nodes[self.L], z
def get_data(self, data):
self.answers = np.zeros((len(data), self.LAYER_SIZE[self.L]))
self.inputs = []
for i in range(len(data)):
if (data[i][0] < self.LAYER_SIZE[self.L]):
self.answers[i][data[i][0]] = 1
self.inputs.append(data[i][1])
self.data = [self.answers, self.inputs]
def train(self, iter_num=None, test_data=None):
if (iter_num == None):
iter_num = self.DEFAULT_ITER_NUM
# region: setting variables to be the same as the class variables
total_delta_weights = [
np.zeros((self.LAYER_SIZE[i + 1], self.LAYER_SIZE[i]))
for i in range(self.L)
]
total_delta_bias = [
np.zeros(self.LAYER_SIZE[i + 1]) for i in range(self.L)
]
delta_nodes = list(self.nodes)
delta_bias = list(self.bias)
delta_weights = list(self.weights)
z = list(self.nodes)
delta_z = list(self.nodes)
total_delta_weights = list(self.weights)
total_delta_bias = list(self.bias)
#endregion
# region: setting plot cost vectors
eff_cost_vec = []
for i in range(self.LAYER_SIZE[self.L]):
eff_cost_vec.append([0] * iter_num)
samples_same_ans = [0] * self.LAYER_SIZE[self.L]
max_cost_vec = []
for i in range(self.LAYER_SIZE[self.L]):
max_cost_vec.append([0] * iter_num)
max_vec = []
for i in range(self.LAYER_SIZE[self.L]):
max_vec.append([0 for i in range(self.LAYER_SIZE[self.L])] *
iter_num)
wrong_vec = []
for i in range(self.LAYER_SIZE[self.L]):
wrong_vec.append([0] * iter_num)
# endregion
for iter in range(iter_num):
ef_cost = np.zeros(self.LAYER_SIZE[self.L])
max_cost = np.zeros(self.LAYER_SIZE[self.L])
combined = list(zip(self.inputs, self.answers))
random.shuffle(combined)
inputs, answers = zip(*combined)
for j in range(len(inputs)):
sample = np.array(inputs[j])
answer = np.array(answers[j])
#CALCULATE NODES
a, z = self.run(sample)
ef_cost += np.abs(self.nodes[self.L] - answer) / len(inputs)
max_cost = np.array([
max(m, curr) for m, curr in zip(
max_cost, np.abs(self.nodes[self.L] - answer))
])
#LEARN
delta_z[self.L] = cost(self.nodes[self.L], answer,
True) * activation_func(
z[self.L], True)
for i in range(1, self.L + 1):
delta_bias[self.L - i] = np.array(delta_z[self.L + 1 - i])
delta_weights[self.L - i] = np.outer(
delta_z[self.L + 1 - i], self.nodes[self.L - i])
delta_nodes[self.L - i] = np.dot(
delta_z[self.L + 1 - i],
delta_weights[self.L - i]) / self.LAYER_SIZE[self.L -
i + 1]
delta_z[self.L -
i] = delta_nodes[self.L - i] * activation_func(
z[self.L - i], True)
if (j == 1 or j == len(inputs) - 1):
self.debug.log("j=" + str(j) + ", " +
str(self.nodes[self.L]))
self.debug.log("answer = " + str(answer))
total_delta_weights = [
np.nan_to_num(tdw + dw)
for tdw, dw in zip(total_delta_weights, delta_weights)
]
total_delta_bias = [
np.nan_to_num(tdb + db)
for tdb, db in zip(total_delta_bias, delta_bias)
]
# region: create vector of results to plot
for i in range(len(answer)):
if (answer[i] == 1):
# self.debug.log("in input j: " + str(j) + " the answer is " + str(
# answer) + " the net answer is " + str(self.nodes[self.L]) +
# " adding cost to place " + str(i) + " " + str(iter))
if (iter == 0):
samples_same_ans[i] += 1
eff_cost_vec[i][iter] += (
(sum(np.abs(self.nodes[self.L] - answer)) /
self.LAYER_SIZE[self.L])**(1 / 2))
if (max_cost_vec[i][iter] < sum(
np.abs(self.nodes[self.L] - answer))):
max_cost_vec[i][iter] = sum(
np.abs(self.nodes[self.L] - answer))
max_vec[i][iter] = self.nodes[self.L]
if (sum(cost(self.nodes[self.L], answer)) > 0.1):
wrong_vec[i][iter] += 1
# endregion
self.weights = [
np.nan_to_num(w - self.DAMP * dw / len(inputs))
for w, dw in zip(self.weights, total_delta_weights)
]
self.bias = [
np.nan_to_num(b - self.DAMP * db / len(inputs))
for b, db in zip(self.bias, total_delta_bias)
]
#self.debug.log("TOTAL DELTA="+str(DAMP*total_delta_weights[self.L-1] / len(inputs)))
# normalize by num of samples
for i in range(len(eff_cost_vec)):
eff_cost_vec[i][
iter] = eff_cost_vec[i][iter] / samples_same_ans[i]
# endregion
self.debug.log(max_cost_vec[0][iter])
self.debug.log("i=0, max=" + str(max_vec[0][iter]))
# region: plot graphs
iter_vec = []
for i in range(len(eff_cost_vec)):
iter_vec.append(np.arange(len(eff_cost_vec[i])))
simplePlot(iter_vec, eff_cost_vec, self.DAMP)
simplePlot(iter_vec, wrong_vec, self.DAMP)
# endregion
return self.weights, self.bias, self.LAYER_SIZE
class Network():
# INITIALIZE NEW NETWORK
def __init__(self, damp=0.0001, default_iter_num=10,
network_size=2, layer_size=[18, 1]):
np.set_printoptions(precision=4)
self.debug = Debug()
self.debug.off()
self.DAMP = damp
self.DEFAULT_ITER_NUM = default_iter_num
self.LAYER_SIZE = layer_size
self.NETWORK_SIZE = network_size
self.L = network_size - 1
self.OUTPUT_SIZE = 1
self.nodes = [np.zeros(self.LAYER_SIZE[i]) for i in range(self.L + 1)]
self.weights = [0.01 * (np.random.rand(self.LAYER_SIZE[i + 1], self.LAYER_SIZE[i]) - 0.5) for i in range(self.L)]
self.bias = [0.01 * (np.random.rand(self.LAYER_SIZE[i + 1]) - 0.5) for i in range(self.L)]
self.network_train_counter = 0
def save(self, filename):
data = {
"weights" : array_to_list(self.weights),
"bias" : array_to_list(self.bias)
}
with open(filename, 'w') as file:
json.dump(data, file, indent=4)
def load(self, filename):
data = {}
with open(filename, 'r') as file:
data = json.load(file)
w = data["weights"]
b = data["bias"]
self.weights = [np.array(w[i]) for i in range(len(w))]
self.bais = [np.array(b[i]) for i in range(len(b))]
def run(self, sample):
if(len(sample) != len(self.nodes[0])):
print("ERROR - sample length does not match first layer size.\nSample: "+str(sample))
self.nodes[0] = activation_func(sample)
#self.nodes[0] = sample
z = list(self.nodes)
for i in range(1, self.L + 1):
z[i] = np.dot(self.weights[i - 1], self.nodes[i - 1]) + self.bias[i - 1]
self.nodes[i] = activation_func(z[i])
return self.nodes[self.L], z
def get_data(self, data):
self.answers = np.zeros(len(data))
self.inputs = []
for i in range(len(data)):
self.answers[i] = 1 - data[i][0]
self.inputs.append(data[i][1])
self.data = [self.answers, self.inputs]
def cal_samples(self, answers):
samples_same_ans = [0, 0]
for j in range(len(answers)):
answer = np.array(answers[j])
samples_same_ans[1 - int(answer)] += 1
return samples_same_ans
def train(self, iter_num=None, test_data=None):
testing = not test_data is None
if testing:
test_inputs = [test_data[j][1] for j in range(len(test_data))]
test_answers = [1 - test_data[j][0] for j in range(len(test_data))]
if(iter_num == None):
iter_num = self.DEFAULT_ITER_NUM
# region: setting variables to be the same as the class variables
total_delta_weights = [np.zeros((self.LAYER_SIZE[i + 1], self.LAYER_SIZE[i])) for i in range(self.L)]
total_delta_bias = [np.zeros(self.LAYER_SIZE[i + 1]) for i in range(self.L)]
delta_nodes = list(self.nodes)
delta_bias = list(self.bias)
delta_weights = list(self.weights)
z = list(self.nodes)
delta_z = list(self.nodes)
total_delta_weights = list(self.weights)
total_delta_bias = list(self.bias)
#endregion
# region: setting plot cost vectors
eff_cost_vec = [[0] * iter_num for i in range(self.LAYER_SIZE[self.L] + 1)]
test_cost_vec = list(eff_cost_vec)
wrong_vec = [0] * iter_num
samples_same_ans = self.cal_samples(self.answers)
tests_same_ans = self.cal_samples(test_answers)
# endregion
for iter in range(iter_num):
combined = list(zip(self.inputs, self.answers))
random.shuffle(combined)
inputs, answers = zip(*combined)
for j in range(len(inputs)):
sample = np.array(inputs[j])
answer = answers[j]
#CALCULATE NODES
a, z = self.run(sample)
#LEARN
delta_z[self.L] = cost(self.nodes[self.L], answer, True) * activation_func(z[self.L], True)
for i in range(1, self.L + 1):
delta_bias[self.L - i] = np.array(delta_z[self.L + 1 - i])
delta_weights[self.L - i] = np.outer(delta_z[self.L + 1 - i], self.nodes[self.L - i])
delta_nodes[self.L - i] = np.dot(delta_z[self.L + 1 - i], delta_weights[self.L - i]) / self.LAYER_SIZE[self.L - i + 1]
delta_z[self.L - i] = delta_nodes[self.L - i] * activation_func(z[self.L - i], True)
if(j == 1 or j == len(inputs)-1):
self.debug.log("j="+str(j)+", "+str(self.nodes[self.L]))
self.debug.log("answer = "+str(answer))
total_delta_weights = [np.nan_to_num(tdw + dw) for tdw, dw in zip(total_delta_weights, delta_weights)]
total_delta_bias = [np.nan_to_num(tdb + db) for tdb, db in zip(total_delta_bias, delta_bias)]
eff_cost_vec[1-int(answer)][iter] += sum(np.abs(self.nodes[self.L] - answer))
if(identify(self.nodes[self.L]) == answer):
wrong_vec[iter] += 1
for j in range(len(test_inputs)):
sample = np.array(test_inputs[j])
answer = test_answers[j]
a, z = self.run(sample)
test_cost_vec[1-int(answer)][iter] += sum(np.abs(self.nodes[self.L] - answer))
if(identify(self.nodes[self.L]) == answer):
wrong_vec[iter] += 1
# endregion
self.weights = [np.nan_to_num(w - self.DAMP * dw / len(inputs)) for w, dw in zip(self.weights, total_delta_weights)]
self.bias = [np.nan_to_num(b - self.DAMP * db / len(inputs)) for b, db in zip(self.bias, total_delta_bias)]
#self.debug.log("TOTAL DELTA="+str(DAMP*total_delta_weights[self.L-1] / len(inputs)))
# normalize by num of samples
for i in range(len(eff_cost_vec)):
eff_cost_vec[i][iter] = eff_cost_vec[i][iter] / samples_same_ans[i]
# endregion
# region: plot graphs
iter_vec = []
for i in range(len(eff_cost_vec)):
iter_vec.append(np.arange(len(eff_cost_vec[i])))
if testing:
y = np.array(eff_cost_vec[0])
y2 = np.array(eff_cost_vec[1])
y3 = np.array(wrong_vec)
plot3(iter_vec[0], y, y2, y3, "Iterations", "Cost", str(self.DAMP))
y = np.array(test_cost_vec[0])
y2 = np.array(test_cost_vec[1])
plot3(iter_vec[0], y, y2, y3, "Iterations", "Cost", str(self.DAMP))
# endregion
return self.weights, self.bias, self.LAYER_SIZE
