/
identity.py
162 lines (126 loc) · 5.92 KB
/
identity.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
# identity.py
# test/retest program results --> to benchmark other versions of the code...
#TODO implement plot (or just simple display) of netinputs for all examples
#TODO implement weight-sharing
from __future__ import division
import math, random
import numpy as np
import matplotlib.pyplot as plt
from pandas import Series, DataFrame
import pandas as pd
from jorg.neuralnet_v1 import \
NeuralNet, Instance, NetworkDataCollector, weight_init_function_random, learning_rate_function
from jorg.activation_classes import SigmoidIO, LinearIO, ConstantOutput, GaussGauss, Gauss, STDNonMonotonicIOFunction
n_hidden_neurons = 1
learning_rate = -0.01
rotation = 0.0
n_trials = 10
max_epochs = 10000
error_criterion = 0.00001
def learning_rate_function():
return learning_rate
def experiment_set_selected_weights(network):
# output_neurons = network.layers[2].neurons
# for output_neuron in output_neurons:
# output_neuron.links[0].weight = 0.0
# output_neuron.links[1].weight = 0.0
hidden_neurons = network.layers[1].neurons_wo_bias_neuron
for hidden_neuron in hidden_neurons:
hidden_neuron.links[0].weight = 2.0
hidden_neuron.links[1].weight = 0.0
# "identity" training set
training_set = [ Instance( [0.0, 0.0], [0.0, 0.0] ), Instance( [0.0, 1.0], [0.0, 1.0] ), Instance( [1.0, 0.0], [1.0, 0.0] ), Instance( [1.0, 1.0], [1.0, 1.0] ) ]
def calc_n_hidden_layers(n_neurons_for_each_layer):
n_hidden_layers = 0
if len(n_neurons_for_each_layer) > 2:
n_hidden_layers = len(n_neurons_for_each_layer) - 2
return n_hidden_layers
def intermediate_post_process_weights(trial_params, data_collector, dfs_concatenated):
weight_series = data_collector.extract_weights(layer_number=1)
#print "weight_series =", weight_series
w0_series = weight_series[:,0,0]
w0_series.name = "weight0"
w1_series = weight_series[:,0,1]
w1_series.name = "weight1"
ratios = w0_series / w1_series
convert_to_angles_function = lambda x: 180.0 * math.atan(x) / math.pi
extracted_series = ratios.map(convert_to_angles_function)
extracted_series.name = "hyperplane_angle"
df = DataFrame([w0_series, w1_series, extracted_series])
#, columns=["weight0", "weight1", "hyperplane_angle", "epochs"] )
df["treatment"] = trial_params
dfs_concatenated = pd.concat([dfs_concatenated, df])
return dfs_concatenated
def intermediate_post_process_netinputs(trial_params, data_collector, dfs_concatenated):
weight_series = data_collector.extract_netinputs(layer_number=1)
#print "weight_series =", weight_series
w0_series = weight_series[:,0,0]
w0_series.name = "weight0"
w1_series = weight_series[:,0,1]
w1_series.name = "weight1"
ratios = w0_series / w1_series
convert_to_angles_function = lambda x: 180.0 * math.atan(x) / math.pi
extracted_series = ratios.map(convert_to_angles_function)
extracted_series.name = "hyperplane_angle"
df = DataFrame([w0_series, w1_series, extracted_series])
#, columns=["weight0", "weight1", "hyperplane_angle", "epochs"] )
df["treatment"] = trial_params
dfs_concatenated = pd.concat([dfs_concatenated, df])
return dfs_concatenated
# rotate clockwise!!
for an_instance in training_set:
an_instance.rotate_by(rotation)
n_inputs = 2
n_outputs = 2
n_neurons_for_each_layer = [n_inputs, n_hidden_neurons, n_outputs]
n_hidden_layers = calc_n_hidden_layers(n_neurons_for_each_layer)
sigmoid = SigmoidIO()
linear = LinearIO()
constant = ConstantOutput()
nonmon = STDNonMonotonicIOFunction()
# specify neuron transforms, weight initialization, and learning rate functions... per layer
neurons_ios = [None] + [nonmon] * n_hidden_layers + [linear]
weight_init_functions = [None] + [ weight_init_function_random ]*n_hidden_layers + [ weight_init_function_random ]
learning_rate_functions = [None] + [ learning_rate_function ]*n_hidden_layers + [ learning_rate_function ]
results = []
dfs_concatenated = DataFrame([])
for seed_value in range(n_trials):
print "seed = ", seed_value,
random.seed(seed_value)
# initialize the neural network
network = NeuralNet(n_neurons_for_each_layer, neurons_ios, weight_init_functions, learning_rate_functions)
experiment_set_selected_weights(network)
print "\n\nNet BEFORE Training\n", network
data_collection_interval = 1000
data_collector = NetworkDataCollector(network, data_collection_interval)
# start training on test set one
epoch_and_MSE = network.backpropagation(training_set, error_criterion, max_epochs, data_collector) # sop call
# epoch_and_MSE = network.backpropagation(training_set, 0.0000001, max_epochs, data_collector)
results.append(epoch_and_MSE[0])
#print "\n\nNet After Training\n", network
# save the network
network.save_to_file( "trained_configuration.pkl" )
# load a stored network
# network = NeuralNet.load_from_file( "trained_configuration.pkl" )
dfs_concatenated = intermediate_post_process_weights(seed_value, data_collector, dfs_concatenated)
# print out the result
for example_number, example in enumerate(training_set):
inputs_for_training_example = example.features
network.inputs_for_training_example = inputs_for_training_example
output_from_network = network.calc_networks_output()
print "\tnetworks input:", example.features, "\tnetworks output:", output_from_network, "\ttarget:", example.targets
print results
print
print np.median(results)
print
print dfs_concatenated
print
end_angle_values = dfs_concatenated["end"]["hyperplane_angle"]
treatment_values = dfs_concatenated["treatment"]["hyperplane_angle"]
list_of_dfs = [treatment_values] + [ (dfs_concatenated[epochs]["hyperplane_angle"]) for epochs in [0] ] + [end_angle_values]
selected_df = pd.concat( list_of_dfs, axis=1 )
print selected_df
print type(selected_df)
plt.scatter(selected_df["treatment"], selected_df["end"])
#pd.scatter_matrix(selected_df, diagonal='kde', color='k', alpha=0.3)
plt.show()