def getOutput(self, new_inputs):
# calculate the weighted linear sum
sum = ms.weightedSum(new_inputs, self.weights, len(new_inputs))
# if a logistic unit, return the logistic(sum)
if self.is_logistic:
return ms.logistic(sum)
else:
return sum
def tune(self, input_data, validation_data):
print("Tuning RBF Network")
eta = 0.05
lowest_eta = -1
lowest_error = -1
print("Tuning RBF eta")
while eta <= 0.5:
self.trainOutputLayer(input_data, eta, 0, 10)
error = 0
if self.uses_regression:
# get absolute error for test
results = ms.testRegressor(self, validation_data)
for obs in range(len(results)):
error += (results[obs] * results[obs])
error /= len(results)
else:
error = self.testClassification(validation_data)
print("MSE for eta =", eta, ":", error, "lowest MSE =",
lowest_error)
if (error < lowest_error) or (eta == 0.05):
lowest_eta = eta
lowest_error = error
eta += 0.05
for node in range(len(self.output_layer)):
self.output_layer[node].resetWeights()
print("Selected RBF eta =", lowest_eta)
#self.trainOutputLayer(input_data, eta, 0)
print("Tuning RBF alpha for momentum")
alpha = 0
lowest_alpha = 0
lowest_error = -1
while alpha < 0.5:
self.trainOutputLayer(input_data, lowest_eta, alpha, 10)
error = 0
if self.uses_regression:
# get absolute error for each validation observation
results = ms.testRegressor(self, validation_data)
for obs in range(len(results)):
error += (results[obs] * results[obs])
error /= len(results)
else:
error = self.testClassification(validation_data)
print("MSE for alpha =", alpha, ":", error, "lowest MSE =",
lowest_error)
if (error < lowest_error) or (alpha == 0):
lowest_alpha = alpha
lowest_error = error
prev_error = error
alpha += 0.1
for node in range(len(self.output_layer)):
self.output_layer[node].resetWeights()
print("Selected RBF alpha =", lowest_alpha)
now = time.time()
self.trainOutputLayer(input_data, lowest_eta, lowest_alpha, 100)
done = time.time()
self.convergence_time = done - now
def openFile(data_file):
lines = open(data_file, "r").readlines()
csv_lines = reader(lines)
data = []
for line in csv_lines:
tmp = []
if sys.argv[2] == 'rm':
# remove line number from each example (first column)
for c in range(1, len(line) - 1):
tmp.append(float(line[c]))
else:
for c in range(len(line) - 1):
tmp.append(float(line[c]))
if sys.argv[3] == 'r':
tmp.append(float(line[-1]))
else:
tmp.append(line[-1])
data.append(tmp)
data = ms.normalize(data)
# divide data into 10 chunks for use in 10-fold cross validation paired t test
chnks = getNChunks(data, 10)
class_list = getClasses(data)
return chnks, class_list
def getNeighbors(self, new_obs):
#print("Finding nearest Neighbors")
# dists: an array of tuples for every item in the training set of the form (training set obs, dist to new obs)
dists = []
for x in range(len(self.training_set)):
if self.similarity_type == 'mass':
dist = self.alt_similarity.getMassDissimilarity(
new_obs, self.training_set[x])
else:
dist = ms.squaredDistance(new_obs, self.training_set[x],
len(self.training_set[0]) - 1)
# Append the observation from the training set and its distance as a tuple to the distances array
dists.append((self.training_set[x], dist))
# sort method uses a key function to be applied to objects to be sorted, so I use this lambda function to tell it
# to sort by the element at index 1 of the tuple (the distance)
dists.sort(key=lambda elem: elem[1])
neighbors = []
# Now that dists is sorted by distance, the first k elements are the k nearest neighbors
# unless there are less than k neighbors
if len(self.training_set) < self.k:
num_neighbors = len(self.training_set)
else:
num_neighbors = self.k
for x in range(num_neighbors):
# We don't need the distance value anymore, so we only append the training set obs (pull item at index 0
# from the tuple
neighbors.append(dists[x][0])
return neighbors
def predict(self, new_obs):
# dists: an array of tuples for every item in the training set of the form (training set obs, dist to new obs)
dists = []
for x in range(len(self.medoids)):
dist = ms.squaredDistance(new_obs, self.medoids[x], len(self.medoids[0]) - 1)
# Append the observation from the medoid list and its distance as a tuple to the distances array
dists.append((self.medoids[x], dist))
# sort method uses a key function to be applied to objects to be sorted, so I use this lambda function to tell it
# to sort by the element at index 1 of the tuple (the distance)
dists.sort(key=lambda elem: elem[1])
# [closest tuple][first elem of tuple (medoid)][last elem of medoid]
return dists[0][0][-1]
def __init__(self, data, k, uses_regression, min_examples_in_cluster):
print("Finding centroids")
self.centroids = [[]] * k
self.clust = []
# randomize original data
random.shuffle(data)
for i in range(k - 1, -1, -1):
# randomly initialize centroids to values in the data set (not required to be values in the data set, this is supposed to make it converge faster)
self.centroids[i] = copy.deepcopy(data[i])
del self.centroids[i][-1]
oldState = [
] #We first define an oldState variable to remember the place the centroids were in
while (self.centroidsMoved(oldState)):
oldState = copy.deepcopy(
self.centroids
) #if we get to the start of the loop, we'll need to deep copy the last state of Centroids for later
#We reset the Clusters 2 steps, a clear, then we fill it with k empty indices
self.clust = []
for i in range(k):
self.clust.append([])
for xi in range(len(data)):
#Clustering starts here. Let's split each observation where it needs to go.
closestCentroid = self.centroids[
0] #We autodefine the first centroid as the closest point for now
closestDistance = ms.squaredDistance(self.centroids[0],
data[xi],
len(self.centroids[0]))
for y in range(1, len(
self.centroids)): #We now look for actual the Argmin
tempDistance = ms.squaredDistance(self.centroids[y],
data[xi],
len(self.centroids[y]))
if tempDistance < closestDistance:
closestDistance = tempDistance
closestCentroid = self.centroids[y]
#At the end of the loop, when the closest Centroid has been defined, stick the data line to the closest Cluster
self.clust[self.centroids.index(closestCentroid)].append(
copy.deepcopy(data[xi]))
#Recalculating Centroids now.
# reset centroids to 0
for centroid_num in range(len(self.centroids)):
for feature_num in range(len(self.centroids[0])):
self.centroids[centroid_num][feature_num] = 0
# sum elements from clusters into centroids and divide: centroids will be the average coordinates of obs in their cluster
for centroid_num in range(len(self.centroids)):
for clust_obs in range(len(self.clust[centroid_num])):
for feature_num in range(len(self.centroids[0])):
self.centroids[centroid_num][
feature_num] += self.clust[centroid_num][
clust_obs][feature_num]
for feature_num in range(len(self.centroids[0])):
if len(self.clust[centroid_num]) != 0:
self.centroids[centroid_num][feature_num] /= len(
self.clust[centroid_num])
for centroid_num in range(len(self.centroids) - 1, -1, -1):
if (len(self.clust[centroid_num]) < min_examples_in_cluster) or (
len(self.clust[centroid_num]) == 0):
del self.clust[centroid_num]
del self.centroids[centroid_num]
continue
if uses_regression:
self.centroids[centroid_num].append(self.regress(centroid_num))
else:
self.centroids[centroid_num].append(
self.classify(centroid_num))
def tenFoldCV(chunked_data, clss_list, use_regression, k, k2, t, phi, kc, kc2,
ct, cphi, ke, ke2, et, ephi):
knn_missed = []
knn2_missed = []
cnn_missed = []
cnn2_missed = []
enn_missed = []
enn2_missed = []
for test_num in range(10):
print("\n\n\nFold: ", test_num + 1, "of 10 fold cross validation")
training_set = []
testing_set = chunked_data[test_num]
# make example set
for train in range(10):
if train != test_num:
for x in range(len(chunked_data[train])):
training_set.append(chunked_data[train][x])
validation_index = int((float(len(training_set)) * 8 / 10)) - 1
knn = nn.NearestNeighbor(training_set, k)
#knn.massSimilarity(int(sys.argv[16]),int(sys.argv[17]))
knn2 = nn.NearestNeighbor(training_set, k2)
knn2.massSimilarity(t, phi)
knn_missed.append(ms.testClassifier(knn, testing_set))
knn2_missed.append(ms.testClassifier(knn2, testing_set))
cnn = nn.NearestNeighbor(training_set, kc)
cnn.convertToCondensed()
#cnn.massSimilarity(int(sys.argv[18]),int(sys.argv[19]))
cnn2 = nn.NearestNeighbor(training_set, kc2)
cnn2.convertToCondensed()
cnn2.massSimilarity(ct, cphi)
cnn_missed.append(ms.testClassifier(cnn, testing_set))
cnn2_missed.append(ms.testClassifier(cnn2, testing_set))
enn = nn.NearestNeighbor(training_set[:validation_index], ke)
enn.convertToEdited(training_set[validation_index:])
#enn.massSimilarity(int(sys.argv[20]),int(sys.argv[21]))
enn2 = nn.NearestNeighbor(training_set[:validation_index], ke2)
enn2.convertToEdited(training_set[validation_index:])
enn2.massSimilarity(et, ephi)
enn_missed.append(ms.testClassifier(enn, testing_set))
enn2_missed.append(ms.testClassifier(enn2, testing_set))
ms.compareClassifiers(knn_missed, knn2_missed, 'knn', 'knn2')
ms.compareClassifiers(knn_missed, cnn_missed, 'knn', 'cnn')
ms.compareClassifiers(knn_missed, cnn2_missed, 'knn', 'cnn2')
ms.compareClassifiers(knn_missed, enn_missed, 'knn', 'enn')
ms.compareClassifiers(knn_missed, enn2_missed, 'knn', 'enn2')
ms.compareClassifiers(knn2_missed, cnn_missed, 'knn2', 'cnn')
ms.compareClassifiers(knn2_missed, cnn2_missed, 'knn2', 'cnn2')
ms.compareClassifiers(knn2_missed, enn_missed, 'knn2', 'enn')
ms.compareClassifiers(knn2_missed, enn2_missed, 'knn2', 'enn2')
ms.compareClassifiers(cnn_missed, cnn2_missed, 'cnn', 'cnn2')
ms.compareClassifiers(cnn_missed, enn_missed, 'cnn', 'enn')
ms.compareClassifiers(cnn_missed, enn2_missed, 'cnn', 'enn2')
ms.compareClassifiers(cnn2_missed, enn_missed, 'cnn2', 'enn')
ms.compareClassifiers(cnn2_missed, enn2_missed, 'cnn2', 'enn2')
ms.compareClassifiers(enn_missed, enn2_missed, 'enn', 'enn2')
def __init__(self, data, k, uses_regression, min_examples_in_cluster):
print("Finding medoids")
# appended to using deep copy, holds the medoids
self.medoids = []
# appended to using shallow copy, holds the items clustered around a given medoid
self.clust = []
# randomize original data
random.shuffle(data)
# randomize initial medoids
for i in range(k-1, -1, -1):
# randomly initialize medoids to values in the data set
self.medoids.append(copy.deepcopy(data[i]))
del data[i]
# initialize clusters to empty arrays
self.clust.append([])
previous_medoids = []
loop_num = 0
# PAM is an iterative approach meaning that even if it isn't run to convergence,
# it is still fairly accurate. Each iteration gives better and better medoids, but
# just like an infinite series: you only need some iterations to get a good approximation.
# Due to this, PAM is infrequently ran to convergence, it is typically stopped after a
# certain number of iterations where more iterations will give slightly more accurate
# results. For the given data sets, I found 6 iterations to be sufficient to give good
# estimates for the medoids.
while self.medoidsMoved(previous_medoids) and (loop_num < 3):
print("Loop", loop_num)
#previous_medoids = copy.deepcopy(self.medoids)
previous_medoids = self.medoids
for x_i in range(len(data)):
# start with considering the first medoid the closest
closest_medoid = self.medoids[0]
closest_distance = ms.squaredDistance(closest_medoid, data[x_i], len(closest_medoid)-1)
# find argmin m_j ( distance(x_i, m_j) )
for m_j in range(1, k):
temp_dist = ms.squaredDistance(self.medoids[m_j], data[x_i], len(self.medoids[m_j])-1)
if temp_dist < closest_distance:
closest_distance = temp_dist
closest_medoid = self.medoids[m_j]
# assign x_i to medoid m_j
self.clust[self.medoids.index(closest_medoid)].append(data[x_i])
# calculate distortion
distor = self.distortion()
# for each m_i in medoids do
for m_i in range(k):
# for each example x_j in data where x_j is not in m_i
for x_j in range(len(data)):
if data[x_j] not in self.clust[m_i]:
# swap m_i and x_j
temp_ex = copy.deepcopy(data[x_j])
data[x_j] = copy.deepcopy(self.medoids[m_i])
self.medoids[m_i] = temp_ex
distor_prime = self.distortion()
# swap back
if distor <= distor_prime:
temp_ex = copy.deepcopy(data[x_j])
data[x_j] = copy.deepcopy(self.medoids[m_i])
self.medoids[m_i] = temp_ex
loop_num += 1
# repeat until no change in medoids (assumming running until convergence) or the specified loop number is reached
for medoid_num in range(len(self.medoids) - 1, -1, -1):
if len(self.clust[medoid_num]) < min_examples_in_cluster:
del self.clust[medoid_num]
del self.medoids[medoid_num]
continue
# add a medoid to its cluster since the medoid is an observation
self.clust[medoid_num].append(self.medoids[medoid_num])
if uses_regression:
#self.medoids[medoids_num].append(self.regress(medoids_num))
self.medoids[medoid_num][-1] = self.regress(medoid_num)
else:
#self.medoids[medoids_num].append(self.classify(medoids_num))
self.medoids[medoid_num][-1] = self.classify(medoid_num)
def tuneLayerwise(self, input_data, eta, alpha_momentum, iterations):
print("Tuning by Layer...")
for layer_num in range(len(self.hidden_layers)):
print("Layer ", layer_num + 1, "of", len(self.hidden_layers))
if layer_num == 0:
inputs = len(input_data[0]) - 1
else:
inputs = len(self.hidden_layers[layer_num - 1])
# create outputs for this layer
for output_node in range(inputs):
self.output_layer.append(
unit.Neuron(len(self.hidden_layers[layer_num]),
self.logistic_output))
# tune layer
random.shuffle(input_data)
prev_delta_weights = []
prev_loss = []
worse_epochs = 0
for epoch in range(iterations):
loss = [0.0] * len(self.output_layer)
delta_weights = []
for example_num in range(len(input_data)):
delta_weights.append([])
# get hidden outputs
hidden_outputs = []
for layer in range(layer_num + 1):
if layer == 0:
hidden_outputs.append(
self.getHiddenLayerOutput(
input_data[example_num][:-1], 0))
else:
# else, take outputs from previous layer
hidden_outputs.append(
self.getHiddenLayerOutput(
hidden_outputs[-1], layer))
# get outputs
outputs = []
#if hidden_outputs == []:
# hidden_outputs.append(input_data[example_num][:-1])
for output_node in range(len(self.output_layer)):
outputs.append(
self.output_layer[output_node].getOutput(
hidden_outputs[-1]))
# -------------#
# prev_error[node]
prev_error = []
# prev_weights[node][weight]
prev_weights = []
# append a list for output layer
delta_weights[example_num].append([])
for output_node in range(len(self.output_layer)):
# shallow copy the weights (alias)
weights = self.output_layer[output_node].weights
prev_weights.append(copy.deepcopy(weights))
# append a list to hold the weights for this node
delta_weights[example_num][0].append([])
if not self.regularize:
if layer_num != 0:
error = hidden_outputs[-2][
output_node] - outputs[output_node]
else:
# error = feature val - predicted feature val
error = input_data[example_num][
output_node] - outputs[output_node]
else:
sum_of_weights = 0
for weight_num in range(len(weights)):
sum_of_weights += abs(weights[weight_num])
if layer_num != 0:
error = hidden_outputs[-2][
output_node] - outputs[output_node] + (
self.lmbda * sum_of_weights)
else:
# error = feature val - predicted feature val + lambda*(sum of weights)
error = input_data[example_num][
output_node] - outputs[output_node] + (
self.lmbda * sum_of_weights)
prev_error.append(error)
if layer_num == 0:
loss[output_node] += ms.getDecimalSMAPE(
input_data[example_num][output_node],
outputs[output_node])
else:
loss[output_node] += ms.getDecimalSMAPE(
hidden_outputs[-2][output_node],
outputs[output_node])
for weight_num in range(len(weights)):
delta_weights[example_num][0][output_node].append(
eta * error * hidden_outputs[-1][weight_num])
if (alpha_momentum > 0) and (example_num > 0):
weights[weight_num] = weights[
weight_num] + delta_weights[example_num][
0][output_node][weight_num] + (
alpha_momentum *
prev_delta_weights[0][output_node]
[weight_num])
else:
weights[weight_num] = weights[
weight_num] + delta_weights[example_num][
0][output_node][weight_num]
# propogate errors and update weights
#current_error = []
#current_weights = []
# append a list for hidden layer
delta_weights[example_num].append([])
for node in range(len(self.hidden_layers[layer_num])):
# add list for the weights of this node
delta_weights[example_num][1].append([])
sum_downsteam_error = 0.0
for downstream_node in range(len(prev_error)):
sum_downsteam_error += prev_error[
downstream_node] * prev_weights[
downstream_node][node + 1]
activation = hidden_outputs[-1][node + 1]
if self.logistic_nodes:
current_error = sum_downsteam_error * activation * (
1 - activation)
else:
current_error = sum_downsteam_error
# shallow copy the weights (alias)
weights = self.hidden_layers[layer_num][node].weights
#current_weights.append(copy.deepcopy(weights))
for weight_num in range(len(weights)):
if layer_num != 0:
input = hidden_outputs[-2][weight_num]
else:
if weight_num == 0:
input = 1
else:
input = input_data[example_num][weight_num
- 1]
#if layer_num != 0:
# delta_weights[example_num][1][node].append(eta * current_error * (hidden_outputs[-2][weight_num]))
#else:
# delta_weights[example_num][1][node].append(eta * current_error * (input_data[example_num][weight_num]))
delta_weights[example_num][1][node].append(
eta * current_error * input)
if (alpha_momentum > 0) and (example_num > 0):
weights[weight_num] = weights[
weight_num] + delta_weights[example_num][
1][node][
weight_num] + alpha_momentum * prev_delta_weights[
1][node][weight_num]
else:
weights[weight_num] = weights[
weight_num] + delta_weights[example_num][
1][node][weight_num]
#prev_error = current_error
#prev_weights = current_weights
if alpha_momentum > 0:
prev_delta_weights = delta_weights[-1]
print(self.name, ":", epoch + 1, "of", iterations, end=' ')
average = 0.0
for output_num in range(len(self.output_layer)):
average += loss[output_num]
# take average and convert decimal to %
average /= len(input_data)
average *= (100 / len(self.output_layer))
if self.regularize:
print("Regularized ", end='')
print("Average Symmetric Mean Absolute Percentage Error:",
average)
if epoch > 10:
prev_average = 0.0
for output_num in range(len(self.output_layer)):
prev_average += prev_loss[output_num]
# take average and convert decimal to %
prev_average /= len(input_data)
prev_average *= (100 / len(self.output_layer))
if average > prev_average:
worse_epochs += 1
if (worse_epochs > 4) or (average >
(9 * prev_average)):
print("Converged")
break
else:
worse_epochs -= 2
if worse_epochs < 0:
worse_epochs = 0
prev_loss = loss
# clear outputlayer
self.output_layer = []
def backpropogation(self, input_data, hidden_layer_nodes, eta,
alpha_momentum, iterations):
print("Training", self.name, "using: eta =", eta, ", alpha =",
alpha_momentum, ", nodes by layer =", hidden_layer_nodes)
for layer in range(len(hidden_layer_nodes)):
if layer == 0:
# if first hidden layer, number of inputs is number of features
inputs = len(input_data[0]) - 1
else:
# else number of inputs is number of nodes from previous layer (bias added automatically)
inputs = len(self.hidden_layers[-1])
self.hidden_layers.append([])
for node in range(hidden_layer_nodes[layer]):
self.hidden_layers[layer].append(
unit.Neuron(inputs, self.logistic_nodes))
if (self.output_type
== "autoencoder") and (len(hidden_layer_nodes) > 1):
self.tuneLayerwise(input_data, eta, alpha_momentum, iterations)
print("Fine tuning weights using Backpropogation")
#-create output nodes-#
# if there is at least one hidden layer, set number of inputs to the number of nodes in the final layer
if len(hidden_layer_nodes) != 0:
inputs = len(self.hidden_layers[-1])
else:
# set inputs equal to number of features of input data
inputs = len(input_data[0]) - 1
for output_node in range(self.out_k):
self.output_layer.append(unit.Neuron(inputs, self.logistic_output))
if not ((self.output_type == "regression") or
(self.output_type == "autoencoder")):
self.output_layer[output_node].setClass(
self.class_list[output_node])
#-----------------#
# Backpropogation #
#-----------------#
random.shuffle(input_data)
prev_delta_weights = []
prev_loss = []
worse_epochs = 0
for epoch in range(iterations):
loss = [0.0] * self.out_k
# delta_weights[example_num][layer][node][weight]
delta_weights = []
for example_num in range(len(input_data)):
delta_weights.append([])
#-------------#
#-Get Outputs-#
# -------------#
# get the output for each hidden node of each hidden layer
# hidden_outputs[layer][node]
hidden_outputs = []
if len(self.hidden_layers) > 0:
for layer in range(len(self.hidden_layers)):
# if first layer, take data inputs
if layer == 0:
hidden_outputs.append(
self.getHiddenLayerOutput(
input_data[example_num][:-1], 0))
else:
# else, take outputs from previous layer
hidden_outputs.append(
self.getHiddenLayerOutput(
hidden_outputs[-1], layer))
# there are no hidden layers
else:
# add bias node
hidden_outputs.append([1] + input_data[example_num][:-1])
# get outputs by layer
# outputs [layer][node]
outputs = []
for output_node in range(self.out_k):
outputs.append(self.output_layer[output_node].getOutput(
hidden_outputs[-1]))
# -------------#
#-get output node errors and update output weights-#
#--------------------------------------------------#
# prev_error[node]
prev_error = []
# prev_weights[node][weight]
prev_weights = []
# append a list for output layer
delta_weights[example_num].append([])
for output_node in range(self.out_k):
# shallow copy the weights (alias)
weights = self.output_layer[output_node].weights
prev_weights.append(copy.deepcopy(weights))
# append a list to hold the weights for this node
delta_weights[example_num][0].append([])
error = 0
if self.output_type == "regression":
error = input_data[example_num][-1] - outputs[
output_node]
loss[output_node] += error * error
elif self.output_type == "classification":
if input_data[example_num][-1] == self.output_layer[
output_node].clss:
correct = 1
else:
correct = 0
error = correct - outputs[output_node]
error *= outputs[output_node] * (1 -
outputs[output_node])
loss[output_node] += error * error
elif self.output_type == "autoencoder":
if not self.regularize:
# error = feature val - predicted feature val
error = input_data[example_num][
output_node] - outputs[output_node]
else:
sum_of_weights = 0
for weight_num in range(len(weights)):
sum_of_weights += abs(weights[weight_num])
# error = feature val - predicted feature val + lambda*(sum of weights)
error = input_data[example_num][
output_node] - outputs[output_node] + (
self.lmbda * sum_of_weights)
loss[output_node] += ms.getDecimalSMAPE(
input_data[example_num][output_node],
outputs[output_node])
prev_error.append(error)
for weight_num in range(len(weights)):
delta_weights[example_num][0][output_node].append(
eta * error * hidden_outputs[-1][weight_num])
if (alpha_momentum > 0) and (example_num > 0):
weights[weight_num] = weights[
weight_num] + delta_weights[example_num][0][
output_node][weight_num] + (
alpha_momentum * prev_delta_weights[0]
[output_node][weight_num])
else:
weights[weight_num] = weights[
weight_num] + delta_weights[example_num][0][
output_node][weight_num]
# propogate errors and update weights
# start at final layer and move backwards
prev_layer_error = []
for layer in range(len(self.hidden_layers)):
delta_weights[example_num].append([])
for layer in range(len(self.hidden_layers) - 1, -1, -1):
# check if the downstream layer is the outputs
if layer == (len(self.hidden_layers) - 1):
prev_layer_error = prev_error
current_error = []
current_weights = []
# iterate through the nodes in a given layer
for node in range(len(self.hidden_layers[layer])):
# add list for the weights of this node
delta_weights[example_num][layer + 1].append([])
sum_downsteam_error = 0.0
for downstream_node in range(len(prev_layer_error)):
sum_downsteam_error += prev_layer_error[
downstream_node] * prev_weights[
downstream_node][node + 1]
activation = hidden_outputs[layer][node + 1]
if self.logistic_nodes:
current_error.append(sum_downsteam_error *
activation * (1 - activation))
else:
current_error.append(sum_downsteam_error)
# shallow copy the weights (alias)
weights = self.hidden_layers[layer][node].weights
current_weights.append(copy.deepcopy(weights))
for weight_num in range(len(weights)):
if layer != 0:
input = hidden_outputs[layer - 1][weight_num]
else:
if weight_num == 0:
input = 1
else:
input = input_data[example_num][weight_num
- 1]
delta_weights[example_num][layer + 1][node].append(
eta * current_error[node] * input)
#if layer != 0:
# delta_weights[example_num][layer+1][node].append(eta * current_error[node] * (hidden_outputs[layer-1][weight_num]))
#else:
# delta_weights[example_num][layer+1][node].append(eta * current_error[node] * (input_data[example_num][weight_num]))
if (alpha_momentum > 0) and (example_num > 0):
weights[weight_num] = weights[
weight_num] + delta_weights[example_num][
layer + 1][node][
weight_num] + alpha_momentum * prev_delta_weights[
layer + 1][node][weight_num]
else:
weights[weight_num] = weights[
weight_num] + delta_weights[example_num][
layer + 1][node][weight_num]
prev_layer_error = current_error
prev_weights = current_weights
if alpha_momentum > 0:
prev_delta_weights = delta_weights[-1]
if self.output_type == "regression":
print(self.name, ":", epoch + 1, "of", iterations, end=' ')
print("MSE:", (loss[0] / len(input_data)))
elif self.output_type == "autoencoder":
print(self.name, ":", epoch + 1, "of", iterations, end=' ')
average = 0.0
for output_num in range(len(self.output_layer)):
average += loss[output_num]
# take average and convert decimal to %
average /= len(input_data)
average *= (100 / len(self.output_layer))
if self.regularize:
print("Regularized ", end='')
print("Average Symmetric Mean Absolute Percentage Error:",
average)
else:
print(self.name, ":", epoch + 1, "of", iterations, end=' ')
average = 0.0
for output_num in range(len(self.output_layer)):
average += loss[output_num] / len(input_data)
average /= len(self.output_layer)
print("Average MSE:", average)
if epoch > 10:
prev_error = 0.0
current_error = 0.0
for output_num in range(len(self.output_layer)):
current_error += loss[output_num]
prev_error += prev_loss[output_num]
if current_error > prev_error:
worse_epochs += 1
if (worse_epochs > 4) or (current_error >
(9 * prev_error)):
print("Converged")
break
else:
worse_epochs -= 2
if worse_epochs < 0:
worse_epochs = 0
prev_loss = loss
def __init__(self, mean, examples):
self.mean = mean
self.variance = ms.getVariance(mean, examples, len(examples[0]) - 1)
def getOutput(self, new_example):
output = 0
output += -1 * ms.squaredDistance(new_example, self.mean,
len(new_example))
output /= (2.0 * pow(self.variance, 2))
return math.exp(output)