def init(self, mins, maxs, num_actions, p):
layers = []
self.state_size = len(list(mins))
self.num_actions = num_actions
self.mins = np.array(mins)
self.maxs = np.array(maxs)
self.incorrect_target = p['incorrect_target']
#print(str(self.state_size) + " " + str(self.num_actions))
self.correct_target = p['correct_target']
layers.append(nnet.layer(self.state_size + self.num_actions))
layers.append(
nnet.layer(p['num_hidden'],
p['activation_function'],
initialization_scheme=p['initialization_scheme'],
initialization_constant=p['initialization_constant'],
dropout=p['dropout'],
use_float32=p['use_float32'],
momentum=p['momentum'],
maxnorm=p['maxnorm'],
step_size=p['learning_rate']))
layers.append(
nnet.layer(
1,
initialization_scheme=p['initialization_scheme_final'],
initialization_constant=p['initialization_constant_final'],
use_float32=p['use_float32'],
momentum=p['momentum'],
step_size=p['learning_rate']))
self.net = nnet.net(layers)
def load_net(filename):
matlabdict = {}
scipy.io.loadmat(filename, matlabdict)
net = nnet
num_layers = matlabdict["num_layers"][0]
layers = [nnet.layer(matlabdict["layer_node_count_input_1"][0])]
for i in range(num_layers):
l = matlabdict["layer_node_count_output_" + str(i + 1)][0]
a = matlabdict["layer_activation_" + str(i + 1)]
layers.append(nnet.layer(l, a))
net = nnet.net(layers)
for i in range(num_layers):
net.layer[i].weights = matlabdict["layer_weights_" + str(i + 1)]
dropout = matlabdict["layer_dropout_" + str(i + 1)][0]
if dropout == "None":
# print('Layer ' + str(i) + ': Dropout is none')
net.layer[i].dropout = None
else:
# print('Layer ' + str(i) + ': Dropout: ' + str(dropout))
net.layer[i].dropout = float(dropout)
data = {}
data["net"] = net
data["sample_mean"] = matlabdict["sample_mean"]
data["sample_std"] = matlabdict["sample_std"]
data["patchsize"] = matlabdict["patchsize"]
data["test_rate"] = matlabdict["test_rate"]
return data
def init(self,state_size,num_actions,p):
layers = [];
self.state_size = state_size
self.num_actions = num_actions
self.incorrect_target = p['incorrect_target']
self.correct_target = p['correct_target']
layers.append(nnet.layer(self.state_size))
if(p.has_key('num_hidden') and p['num_hidden'] is not None):
layers.append(nnet.layer(p['num_hidden'],p['activation_function'],
initialization_scheme=p['initialization_scheme'],
initialization_constant=p['initialization_constant'],
dropout=p['dropout'],use_float32=p['use_float32'],
momentum=p['momentum'],maxnorm=p['maxnorm'],step_size=p['learning_rate']))
layers.append(nnet.layer(num_actions,p['activation_function_final'],
initialization_scheme=p['initialization_scheme_final'],
initialization_constant=p['initialization_constant_final'],
use_float32=p['use_float32'],
momentum=p['momentum'],step_size=p['learning_rate']))
self.net = nnet.net(layers)
self.do_neuron_clustering=False #by default
if(p.has_key('cluster_func') and p['cluster_func'] is not None):
self.net.layer[0].centroids = np.asarray(((np.random.random((self.net.layer[0].weights.shape)) - 0.5) * 2.5),np.float32)
#make the centroid bias input match the bias data of 1.0
self.net.layer[0].centroids[:,-1] = 1.0
self.net.layer[0].select_func = csf.select_names[p['cluster_func']]
print('cluster_func: ' + str(csf.select_names[p['cluster_func']]))
self.net.layer[0].centroid_speed = p['cluster_speed']
self.net.layer[0].num_selected = p['clusters_selected']
self.do_neuron_clustering=True #set a flag to indicate neuron clustering
if(p.has_key('do_cosinedistance') and p['do_cosinedistance']):
self.net.layer[0].do_cosinedistance = True
print('cosine set to true')
def load_net(filename):
matlabdict = {}
scipy.io.loadmat(filename, matlabdict)
net = nnet
num_layers = matlabdict['num_layers'][0]
layers = [nnet.layer(matlabdict['layer_node_count_input_1'][0])]
for i in range(num_layers):
l = matlabdict['layer_node_count_output_' + str(i + 1)][0]
a = matlabdict['layer_activation_' + str(i + 1)]
layers.append(nnet.layer(l, a))
net = nnet.net(layers)
for i in range(num_layers):
net.layer[i].weights = matlabdict['layer_weights_' + str(i + 1)]
dropout = matlabdict['layer_dropout_' + str(i + 1)][0]
if (dropout == 'None'):
#print('Layer ' + str(i) + ': Dropout is none')
net.layer[i].dropout = None
else:
#print('Layer ' + str(i) + ': Dropout: ' + str(dropout))
net.layer[i].dropout = float(dropout)
data = {}
data['net'] = net
data['sample_mean'] = matlabdict['sample_mean']
data['sample_std'] = matlabdict['sample_std']
data['patchsize'] = matlabdict['patchsize']
data['test_rate'] = matlabdict['test_rate']
return data
def init(self, state_size, num_actions, p):
layers = []
self.state_size = state_size
self.num_actions = num_actions
self.incorrect_target = p['incorrect_target']
self.correct_target = p['correct_target']
layers.append(nnet.layer(self.state_size))
if (p.has_key('num_hidden') and p['num_hidden'] is not None):
layers.append(
nnet.layer(
p['num_hidden'],
p['activation_function'],
initialization_scheme=p['initialization_scheme'],
initialization_constant=p['initialization_constant'],
dropout=p['dropout'],
use_float32=p['use_float32'],
momentum=p['momentum'],
maxnorm=p['maxnorm'],
step_size=p['learning_rate']))
layers.append(
nnet.layer(
num_actions,
p['activation_function_final'],
initialization_scheme=p['initialization_scheme_final'],
initialization_constant=p['initialization_constant_final'],
use_float32=p['use_float32'],
momentum=p['momentum'],
step_size=p['learning_rate']))
self.net = nnet.net(layers)
self.do_neuron_clustering = False #by default
if (p.has_key('cluster_func') and p['cluster_func'] is not None):
self.net.layer[0].centroids = np.asarray(((np.random.random(
(self.net.layer[0].weights.shape)) - 0.5) * 2.5), np.float32)
#make the centroid bias input match the bias data of 1.0
self.net.layer[0].centroids[:, -1] = 1.0
self.net.layer[0].select_func = csf.select_names[p['cluster_func']]
print('cluster_func: ' + str(csf.select_names[p['cluster_func']]))
self.net.layer[0].centroid_speed = p['cluster_speed']
self.net.layer[0].num_selected = p['clusters_selected']
self.do_neuron_clustering = True #set a flag to indicate neuron clustering
if (p.has_key('do_cosinedistance') and p['do_cosinedistance']):
self.net.layer[0].do_cosinedistance = True
print('cosine set to true')
def init(self,mins,maxs,num_actions,p):
layers = [];
self.state_size = len(list(mins))
self.num_actions = num_actions
self.mins = np.array(mins)
self.maxs = np.array(maxs)
self.incorrect_target = p['incorrect_target']
#print(str(self.state_size) + " " + str(self.num_actions))
self.correct_target = p['correct_target']
layers.append(nnet.layer(self.state_size + self.num_actions))
layers.append(nnet.layer(p['num_hidden'],p['activation_function'],
initialization_scheme=p['initialization_scheme'],
initialization_constant=p['initialization_constant'],
dropout=p['dropout'],use_float32=p['use_float32'],
momentum=p['momentum'],maxnorm=p['maxnorm'],step_size=p['learning_rate']))
layers.append(nnet.layer(1,
initialization_scheme=p['initialization_scheme_final'],
initialization_constant=p['initialization_constant_final'],
use_float32=p['use_float32'],
momentum=p['momentum'],step_size=p['learning_rate']))
self.net = nnet.net(layers)
def init(self,mins,maxs,num_actions,p):
layers = [];
self.state_size = len(list(mins))
self.num_actions = num_actions
self.mins = np.array(mins)
self.maxs = np.array(maxs)
self.incorrect_target = p['incorrect_target']
print(str(self.state_size) + " " + str(self.num_actions))
self.correct_target = p['correct_target']
layers.append(nnet.layer(self.state_size + self.num_actions))
layers.append(nnet.layer(p['num_hidden'],p['activation_function'],
initialization_scheme=p['initialization_scheme'],
initialization_constant=p['initialization_constant'],
dropout=p['dropout'],use_float32=p['use_float32'],
momentum=p['momentum'],maxnorm=p['maxnorm'],step_size=p['learning_rate']))
layers.append(nnet.layer(1,
initialization_scheme=p['initialization_scheme_final'],
initialization_constant=p['initialization_constant_final'],
use_float32=p['use_float32'],
momentum=p['momentum'],step_size=p['learning_rate']))
self.net = nnet.net(layers)
self.do_neuron_clustering=False #by default
if(p.has_key('cluster_func') and p['cluster_func'] is not None):
#TODO: Make sure the centroids cover the input space appropriately
self.net.layer[0].centroids = np.asarray(((np.random.random((self.net.layer[0].weights.shape)) - 0.5) * 2.25),np.float32)
#make the centroid bias input match the bias data of 1.0
self.net.layer[0].centroids[:,-1] = 1.0
#print(str(self.net.layer[0].centroids.shape))
#print(str(self.net.layer[0].centroids))
self.net.layer[0].select_func = csf.select_names[p['cluster_func']]
#print('cluster_func: ' + str(csf.select_names[p['cluster_func']]))
self.net.layer[0].centroid_speed = p['cluster_speed']
self.net.layer[0].num_selected = p['clusters_selected']
self.do_neuron_clustering=True #set a flag to log neurons that were used for clustering
if(p.has_key('do_cosinedistance') and p['do_cosinedistance']):
self.net.layer[0].do_cosinedistance = True
print('cosine set to true')
(sample_data,class_data) = load_data(range(10),"training",p)
train_size = sample_data.shape[0]
#(test_data,test_class) = load_data(range(10),"testing",p)
#test_size = test_data.shape[0]
num_hidden = p['num_hidden']
training_epochs = p['training_epochs']
minibatch_size = p['minibatch_size']
layers = [];
layers.append(nnet.layer(28*28))
layers.append(nnet.layer(p['num_hidden'],p['activation_function'],select_func=p['select_func'],
select_func_params=p['num_selected_neurons'],
initialization_scheme=p['initialization_scheme'],
initialization_constant=p['initialization_constant'],
dropout=p['dropout'],sparse_penalty=p['sparse_penalty'],
sparse_target=p['sparse_target'],use_float32=p['use_float32'],
momentum=p['momentum'],maxnorm=p['maxnorm'],step_size=p['learning_rate']))
#Add 2nd and 3rd hidden layers if there are parameters indicating that we should
if(p.has_key('num_hidden2')):
layers.append(nnet.layer(p['num_hidden2'],p['activation_function2'],select_func=p['select_func2'],
select_func_params=p['num_selected_neurons2'],
initialization_scheme=p['initialization_scheme2'],
initialization_constant=p['initialization_constant2'],
dropout=p['dropout2'],sparse_penalty=p['sparse_penalty2'],
def init(self,state_size,num_actions,p):
layers = [];
self.state_size = state_size
self.num_actions = num_actions
#self.mins = np.array(mins)
#self.maxs = np.array(maxs)
#self.divs = np.array(divs)
#self.size = self.maxs - self.mins
#self.size = self.size + self.divs
#self.size = self.size/self.divs
#self.arr_mins = (np.zeros(self.size.shape)).astype(np.int64)
#self.arr_maxs = (self.size - np.ones(self.size.shape)).astype(np.int64)
self.incorrect_target = p['incorrect_target']
self.correct_target = p['correct_target']
layers.append(nnet.layer(self.state_size + self.num_actions))
if(p.has_key('num_hidden') and p['num_hidden'] is not None):
layers.append(nnet.layer(p['num_hidden'],p['activation_function'],
initialization_scheme=p['initialization_scheme'],
initialization_constant=p['initialization_constant'],
dropout=p.get('dropout',None),use_float32=p['use_float32'],
momentum=p['momentum'],maxnorm=p.get('maxnorm',None),step_size=p['learning_rate'],rms_prop_rate=p.get('rms_prop_rate',None)))
layers.append(nnet.layer(1,p['activation_function_final'],
initialization_scheme=p['initialization_scheme_final'],
initialization_constant=p['initialization_constant_final'],
use_float32=p['use_float32'],
momentum=p['momentum'],step_size=p['learning_rate'],rms_prop_rate=p.get('rms_prop_rate',None)))
self.net = nnet.net(layers)
self.do_neuron_clustering=False #by default
if(p.has_key('cluster_func') and p['cluster_func'] is not None):
self.cluster_func = p['cluster_func']
self.net.layer[0].centroids = np.asarray(((np.random.random((self.net.layer[0].weights.shape)) - 0.5) * 2.5),np.float32)
#make the centroid bias input match the bias data of 1.0
self.net.layer[0].centroids[:,-1] = 1.0
#print(str(self.net.layer[0].centroids.shape))
#print(str(self.net.layer[0].centroids))
self.net.layer[0].select_func = csf.select_names[p['cluster_func']]
print('cluster_func: ' + str(csf.select_names[p['cluster_func']]))
self.net.layer[0].centroid_speed = p.get('cluster_speed',1.0)
self.net.layer[0].num_selected = p['clusters_selected']
self.do_neuron_clustering=True #set a flag to indicate neuron clustering
if(p.has_key('do_cosinedistance') and p['do_cosinedistance']):
self.net.layer[0].do_cosinedistance = True
print('cosine set to true')
#decay for balancing learning and moving centroids
if(p.has_key('zeta_decay') and p['zeta_decay'] is not None):
self.net.layer[0].zeta_matrix = np.ones(self.net.layer[0].weights.shape,dtype=np.float32)
self.net.layer[0].zeta = 1.0
self.zeta_decay = p['zeta_decay']
self.max_update = 0.0
self.grad_clip = p.get('grad_clip',None)
if(p.has_key('_lambda') and p['_lambda'] is not None):
self._lambda = p['_lambda']
self.gamma = p['gamma']
for l in self.net.layer:
l.eligibility = l.gradient
vx_axis = [p['axis_x_min'],p['axis_x_max']];
vy_axis = [p['axis_y_min'],p['axis_y_max']];
num_classes = p['num_classes']
examples_per_class = p['examples_per_class'];
spread = p['spread']
img_width = p['img_width'];
img_height = p['img_height'];
frameskip = p['frameskip']
num_hidden = p['num_hidden']
layers = [];
layers.append(nnet.layer(2))
layers.append(nnet.layer(p['num_hidden'],p['activation_function'],select_func=p['select_func'],select_func_params=p['num_selected_neurons'],dropout=p['dropout']))
#Add 2nd and 3rd hidden layers if there are parameters indicating that we should
if(p.has_key('num_hidden2')):
layers.append(nnet.layer(p['num_hidden2'],p['activation_function2'],select_func=p['select_func2'],select_func_params=p['num_selected_neurons2'],dropout=p['dropout2']))
if(p.has_key('num_hidden3')):
layers.append(nnet.layer(p['num_hidden3'],p['activation_function3'],select_func=p['select_func3'],select_func_params=p['num_selected_neurons3'],dropout=p['dropout3']))
layers.append(nnet.layer(num_classes,p['activation_function_final']))
learning_rate = p['learning_rate']
#generate random classes
sample_data = np.zeros([2,num_classes*examples_per_class]);
class_data = np.ones([num_classes,num_classes*examples_per_class])*-1.0;
f['centroids'] = centroids
f = h5.File(p['data_dir'] + 'mnist_initial_centroids_' + str(num_centroids) + '.h5py','w')
f.close()
#now we have a k-means clustered set of centroids.
#create an autoencoder network
if(p.has_key('nodes_per_group')):
nodes_per_group=p['nodes_per_group']
else:
nodes_per_group=None
rms_prop_rate = p.get('rms_prop_rate',None)
layers = [];
layers.append(nnet.layer(28*28))
layers.append(nnet.layer(num_centroids,p['activation_function'],
nodes_per_group=nodes_per_group,
initialization_scheme=p['initialization_scheme'],
initialization_constant=p['initialization_constant'],
dropout=p['dropout'],sparse_penalty=p['sparse_penalty'],
sparse_target=p['sparse_target'],use_float32=p['use_float32'],
momentum=p['momentum'],maxnorm=p['maxnorm'],step_size=p['learning_rate']
,rms_prop_rate=rms_prop_rate))
layers.append(nnet.layer(28*28,
initialization_scheme=p['initialization_scheme_final'],
initialization_constant=p['initialization_constant_final'],
use_float32=p['use_float32'],
momentum=p['momentum_final'],step_size=p['learning_rate'],
rms_prop_rate=rms_prop_rate))
def init(self, state_size, num_actions, p):
layers = []
self.do_trig_transform = False
if (p.get('do_trig_transform', True)):
self.do_trig_transform = True
self.state_size = state_size
self.num_actions = num_actions
self.action_dupe_count = p.get('action_dupe_count', 1)
self.do_recurrence = False
if (p['do_recurrence']):
input_size = self.state_size + self.num_actions * self.action_dupe_count + p[
'num_hidden']
self.do_recurrence = True
else:
input_size = self.state_size + self.num_actions * self.action_dupe_count
self.learning_rate = p['learning_rate']
print("state size : " + str(self.state_size))
print("num actions : " + str(self.num_actions))
print("action dupe count : " + str(self.action_dupe_count))
print("num hidden : " + str(p['num_hidden']))
print("input size : " + str(input_size))
self.incorrect_target = p['incorrect_target']
print(str(self.state_size) + " " + str(self.num_actions))
self.correct_target = p['correct_target']
layers.append(nnet.layer(input_size))
layers.append(
nnet.layer(p['num_hidden'],
p['activation_function'],
initialization_scheme=p['initialization_scheme'],
initialization_constant=p['initialization_constant'],
dropout=p['dropout'],
use_float32=p['use_float32'],
momentum=p['momentum'],
maxnorm=p['maxnorm'],
step_size=p['learning_rate']))
if (p.has_key('num_hidden2') and p['num_hidden2'] is not None):
layers.append(
nnet.layer(
p['num_hidden2'],
p['activation_function2'],
initialization_scheme=p['initialization_scheme2'],
initialization_constant=p['initialization_constant2'],
dropout=p['dropout2'],
use_float32=p['use_float32'],
momentum=p['momentum2'],
maxnorm=p['maxnorm2'],
step_size=p['learning_rate2']))
layers.append(
nnet.layer(
1,
initialization_scheme=p['initialization_scheme_final'],
initialization_constant=p['initialization_constant_final'],
use_float32=p['use_float32'],
momentum=p['momentum'],
step_size=p['learning_rate']))
self.net = nnet.net(layers)
if (p.has_key('cluster_func') and p['cluster_func'] is not None):
print("layer 0 has cluster func")
self.cluster_func = p['cluster_func']
#TODO: Make sure the centroids cover the input space appropriately
self.net.layer[0].centroids = np.asarray(((np.random.random(
(self.net.layer[0].weights.shape)) - 0.5) * 2.25), np.float32)
#make the centroid bias input match the bias data of 1.0
self.net.layer[0].centroids[:, -1] = 1.0
#print(str(self.net.layer[0].centroids.shape))
#print(str(self.net.layer[0].centroids))
self.net.layer[0].select_func = csf.select_names[p['cluster_func']]
#print('cluster_func: ' + str(csf.select_names[p['cluster_func']]))
self.net.layer[0].centroid_speed = p['cluster_speed']
self.net.layer[0].num_selected = p['clusters_selected']
if (p.has_key('do_cosinedistance') and p['do_cosinedistance']):
self.net.layer[0].do_cosinedistance = True
print('cosine set to true')
if (p.has_key('zeta_decay') and p['zeta_decay'] is not None):
self.net.layer[0].zeta_matrix = np.ones(
self.net.layer[0].weights.shape, dtype=np.float32)
self.net.layer[0].zeta = 1.0
self.zeta_decay = p['zeta_decay']
if (p.has_key('cluster_func2') and p['cluster_func2'] is not None):
print("layer 1 has cluster func")
self.cluster_func2 = p['cluster_func2']
#TODO: Make sure the centroids cover the input space appropriately
self.net.layer[1].centroids = np.asarray(((np.random.random(
(self.net.layer[1].weights.shape)) - 0.5) * 2.25), np.float32)
#make the centroid bias input match the bias data of 1.0
self.net.layer[1].centroids[:, -1] = 1.0
#print(str(self.net.layer[0].centroids.shape))
#print(str(self.net.layer[0].centroids))
self.net.layer[1].select_func = csf.select_names[
p['cluster_func2']]
#print('cluster_func: ' + str(csf.select_names[p['cluster_func']]))
self.net.layer[1].centroid_speed = p['cluster_speed2']
self.net.layer[1].num_selected = p['clusters_selected2']
if (p.has_key('do_cosinedistance') and p['do_cosinedistance']):
self.net.layer[1].do_cosinedistance = True
print('cosine set to true')
if (p.has_key('zeta_decay2') and p['zeta_decay2'] is not None):
self.net.layer[1].zeta_matrix = np.ones(
self.net.layer[1].weights.shape, dtype=np.float32)
self.net.layer[1].zeta = 1.0
self.zeta_decay = p['zeta_decay2']
self.do_full_zeta = p.get('do_full_zeta', False)
if (p.has_key('_lambda') and p['_lambda'] is not None):
self._lambda = p['_lambda']
self.gamma = p['gamma']
for l in self.net.layer:
l.eligibility = l.gradient
print("Network Size Check:")
print(" weight 0 size: " + str(self.net.layer[0].weights.shape))
if (len(self.net.layer) > 1):
print(" weight 1 size: " + str(self.net.layer[1].weights.shape))
if (len(self.net.layer) > 2):
print(" weight 2 size: " + str(self.net.layer[2].weights.shape))
if (len(self.net.layer) > 3):
print(" weight 3 size: " + str(self.net.layer[3].weights.shape))
#1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
#
#2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
#
#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
#DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
#SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
#THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import time
import numpy as np
#from nnet_toolkit import nnet_cuda as nnet
from nnet_toolkit import nnet
layers = [nnet.layer(2),
nnet.layer(128, 'sigmoid'),
nnet.layer(1, 'sigmoid')]
#layers = [nnet_toolkit.layer(2),nnet_toolkit.layer(256,'linear_rectifier'),nnet_toolkit.layer(128,'linear_rectifier'),nnet_toolkit.layer(64,'linear_rectifier'),nnet_toolkit.layer(32,'linear_rectifier'),nnet_toolkit.layer(1,'squash')];
training_data = np.array([[0, 0, 1, 1], [0, 1, 0, 1]])
training_out = np.array([0, 1, 1, 0])
#net = nnet.net_cuda(layers,step_size=.1);
net = nnet.net(layers, step_size=.1)
net.input = training_data
t = time.time()
for i in range(100000):
net.feed_forward()
net.error = net.output - training_out
centroids = np.array(f['centroids'])
print('centroid data loaded. Shape: ' + str(centroids.shape))
f.close()
#if clusters selected is the same as the number of neurons then we can skip this step (since everything will always be selected)
elif(p['clusters_selected'] != p['num_centroids']):
centroids = do_kmeans(sample_data);
f = h5.File(p['data_dir'] + 'mnist_initial_centroids_' + str(num_centroids) + '.h5py','w')
f['centroids'] = centroids
f.close()
#now we have a k-means clustered set of centroids.
#create a classifier network
layers = [];
layers.append(nnet.layer(input_size))
if(not p.has_key('do_logistic') or p['do_logistic'] == False):
layers.append(nnet.layer(num_centroids,p['activation_function'],
initialization_scheme=p['initialization_scheme'],
initialization_constant=p['initialization_constant'],
dropout=p['dropout'],sparse_penalty=p['sparse_penalty'],
sparse_target=p['sparse_target'],use_float32=p['use_float32'],
momentum=p['momentum'],maxnorm=p['maxnorm'],step_size=p['learning_rate']))
layers.append(nnet.layer(10,p['activation_function_final'],
initialization_scheme=p['initialization_scheme_final'],
initialization_constant=p['initialization_constant_final'],
use_float32=p['use_float32'],
momentum=p['momentum_final'],step_size=p['learning_rate']))
print('total number of samples: ' + str(sample_list.shape[0]))
print('creating testing set')
train_size = sample_list.shape[0]
print('training size: ' + str(sample_list.shape[0]))
print('test size: ' + str(sample_list_test.shape[0]))
class_type = np.sum(class_list, axis=0, dtype=np.float64)
print('class type count: ' + str(class_type))
class_type_test = np.sum(class_list_test, axis=0, dtype=np.float64)
print('class test type count: ' + str(class_type_test))
print('initializing network...')
layers = [nnet.layer(inputsize)]
for i in range(len(hidden_sizes)):
l = hidden_sizes[i]
a = hidden_activations[i]
layers.append(nnet.layer(l, a))
layers.append(nnet.layer(3, 'squash'))
net = nnet.net(layers, step_size=step_size, dropout=dropout_percentage)
print('beginning training...')
save_time = time.time()
epoch_time = time.time()
for i in range(training_epochs):
minibatch_count = int(train_size / minibatch_size)
print('total number of samples: ' + str(sample_list.shape[0]))
print('creating testing set')
train_size = sample_list.shape[0]
print('training size: ' + str(sample_list.shape[0]))
print('test size: ' + str(sample_list_test.shape[0]))
class_type = np.sum(class_list,axis=0,dtype=np.float64)
print('class type count: ' + str(class_type))
class_type_test = np.sum(class_list_test,axis=0,dtype=np.float64)
print('class test type count: ' + str(class_type_test))
print('initializing network...')
layers = [nnet.layer(inputsize)]
for i in range(len(hidden_sizes)):
l = hidden_sizes[i]
a = hidden_activations[i]
layers.append(nnet.layer(l,a))
layers.append(nnet.layer(3,'squash'))
net = nnet.net(layers,step_size=step_size,dropout=dropout_percentage)
print('beginning training...')
save_time = time.time()
epoch_time = time.time()
for i in range(training_epochs):
minibatch_count = int(train_size/minibatch_size)
img_height = p['img_height'];
#the axis for the view
vx_axis = [p['axis_x_min'],p['axis_x_max']];
vy_axis = [p['axis_y_min'],p['axis_y_max']];
num_classes = p['num_classes']
num_hidden = p['num_hidden']
training_epochs = p['training_epochs']
total_epochs = p['total_epochs']
#minibatch_size = p['minibatch_size']
layers = [];
layers.append(nnet.layer(2))
layers.append(nnet.layer(p['num_hidden'],p['activation_function'],
dropout=p['dropout'],sparse_penalty=p['sparse_penalty'],
sparse_target=p['sparse_target'],use_float32=p['use_float32'],
momentum=p['momentum'],maxnorm=p['maxnorm'],step_size=p['learning_rate']))
layers.append(nnet.layer(num_classes,p['activation_function_final'],use_float32=p['use_float32'],
momentum=p['momentum_final'],step_size=p['learning_rate_final']))
#init net
net = nnet.net(layers)
if(p.has_key('cluster_func') and p['cluster_func'] is not None):
#net.layer[0].centroids = np.asarray((((np.random.random((net.layer[0].weights.shape)) - 0.5)*2.0)),np.float32)
net.layer[0].centroids = np.asarray(np.zeros(net.layer[0].weights.shape),np.float32)
#set bias to 1
# #2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. # #THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF #THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import time import numpy as np #from nnet_toolkit import nnet_cuda as nnet from nnet_toolkit import nnet #layers = [nnet_toolkit.layer(2),nnet_toolkit.layer(128,'squash'),nnet_toolkit.layer(1,'squash')]; layers = [nnet.layer(400),nnet.layer(128,'sigmoid'),nnet.layer(3,'sigmoid')]; #training_data = np.array([[0,0,1,1],[0,1,0,1]]); #training_out = np.array([0,1,1,0]); training_data = np.random.random((400,500)); training_out = np.random.random((3,500)); #net = nnet.net_cuda(layers,step_size=.1); net = nnet.net(layers,step_size=.1); net.input = training_data; t = time.time(); for i in range(100000): net.feed_forward(); net.error = net.output - training_out;
def init(self, state_size, num_actions, p):
layers = []
self.state_size = state_size
self.num_actions = num_actions
#self.mins = np.array(mins)
#self.maxs = np.array(maxs)
#self.divs = np.array(divs)
#self.size = self.maxs - self.mins
#self.size = self.size + self.divs
#self.size = self.size/self.divs
#self.arr_mins = (np.zeros(self.size.shape)).astype(np.int64)
#self.arr_maxs = (self.size - np.ones(self.size.shape)).astype(np.int64)
self.incorrect_target = p['incorrect_target']
self.correct_target = p['correct_target']
layers.append(nnet.layer(self.state_size + self.num_actions))
if (p.has_key('num_hidden') and p['num_hidden'] is not None):
layers.append(
nnet.layer(
p['num_hidden'],
p['activation_function'],
initialization_scheme=p['initialization_scheme'],
initialization_constant=p['initialization_constant'],
dropout=p.get('dropout', None),
use_float32=p['use_float32'],
momentum=p['momentum'],
maxnorm=p.get('maxnorm', None),
step_size=p['learning_rate'],
rms_prop_rate=p.get('rms_prop_rate', None)))
layers.append(
nnet.layer(
1,
p['activation_function_final'],
initialization_scheme=p['initialization_scheme_final'],
initialization_constant=p['initialization_constant_final'],
use_float32=p['use_float32'],
momentum=p['momentum'],
step_size=p['learning_rate'],
rms_prop_rate=p.get('rms_prop_rate', None)))
self.net = nnet.net(layers)
self.do_neuron_clustering = False #by default
if (p.has_key('cluster_func') and p['cluster_func'] is not None):
self.cluster_func = p['cluster_func']
self.net.layer[0].centroids = np.asarray(((np.random.random(
(self.net.layer[0].weights.shape)) - 0.5) * 2.5), np.float32)
#make the centroid bias input match the bias data of 1.0
self.net.layer[0].centroids[:, -1] = 1.0
#print(str(self.net.layer[0].centroids.shape))
#print(str(self.net.layer[0].centroids))
self.net.layer[0].select_func = csf.select_names[p['cluster_func']]
print('cluster_func: ' + str(csf.select_names[p['cluster_func']]))
self.net.layer[0].centroid_speed = p.get('cluster_speed', 1.0)
self.net.layer[0].num_selected = p['clusters_selected']
self.do_neuron_clustering = True #set a flag to indicate neuron clustering
if (p.has_key('do_cosinedistance') and p['do_cosinedistance']):
self.net.layer[0].do_cosinedistance = True
print('cosine set to true')
#decay for balancing learning and moving centroids
if (p.has_key('zeta_decay') and p['zeta_decay'] is not None):
self.net.layer[0].zeta_matrix = np.ones(
self.net.layer[0].weights.shape, dtype=np.float32)
self.net.layer[0].zeta = 1.0
self.zeta_decay = p['zeta_decay']
self.max_update = 0.0
self.grad_clip = p.get('grad_clip', None)
if (p.has_key('_lambda') and p['_lambda'] is not None):
self._lambda = p['_lambda']
self.gamma = p['gamma']
for l in self.net.layer:
l.eligibility = l.gradient
def init(self,state_size,num_actions,p):
layers = [];
self.do_trig_transform = False
if(p.get('do_trig_transform',True)):
self.do_trig_transform = True
self.state_size = state_size
self.num_actions = num_actions
self.action_dupe_count = p.get('action_dupe_count',1)
self.do_recurrence = False
if(p['do_recurrence']):
input_size = self.state_size + self.num_actions*self.action_dupe_count + p['num_hidden']
self.do_recurrence = True
else:
input_size = self.state_size + self.num_actions*self.action_dupe_count
self.learning_rate = p['learning_rate']
print("state size : " + str(self.state_size))
print("num actions : " + str(self.num_actions))
print("action dupe count : " + str(self.action_dupe_count))
print("num hidden : " + str(p['num_hidden']))
print("input size : " + str(input_size))
self.incorrect_target = p['incorrect_target']
print(str(self.state_size) + " " + str(self.num_actions))
self.correct_target = p['correct_target']
layers.append(nnet.layer(input_size))
layers.append(nnet.layer(p['num_hidden'],p['activation_function'],
initialization_scheme=p['initialization_scheme'],
initialization_constant=p['initialization_constant'],
dropout=p['dropout'],use_float32=p['use_float32'],
momentum=p['momentum'],maxnorm=p['maxnorm'],step_size=p['learning_rate']))
if(p.has_key('num_hidden2') and p['num_hidden2'] is not None):
layers.append(nnet.layer(p['num_hidden2'],p['activation_function2'],
initialization_scheme=p['initialization_scheme2'],
initialization_constant=p['initialization_constant2'],
dropout=p['dropout2'],use_float32=p['use_float32'],
momentum=p['momentum2'],maxnorm=p['maxnorm2'],step_size=p['learning_rate2']))
layers.append(nnet.layer(1,
initialization_scheme=p['initialization_scheme_final'],
initialization_constant=p['initialization_constant_final'],
use_float32=p['use_float32'],
momentum=p['momentum'],step_size=p['learning_rate']))
self.net = nnet.net(layers)
if(p.has_key('cluster_func') and p['cluster_func'] is not None):
print("layer 0 has cluster func")
self.cluster_func = p['cluster_func']
#TODO: Make sure the centroids cover the input space appropriately
self.net.layer[0].centroids = np.asarray(((np.random.random((self.net.layer[0].weights.shape)) - 0.5) * 2.25),np.float32)
#make the centroid bias input match the bias data of 1.0
self.net.layer[0].centroids[:,-1] = 1.0
#print(str(self.net.layer[0].centroids.shape))
#print(str(self.net.layer[0].centroids))
self.net.layer[0].select_func = csf.select_names[p['cluster_func']]
#print('cluster_func: ' + str(csf.select_names[p['cluster_func']]))
self.net.layer[0].centroid_speed = p['cluster_speed']
self.net.layer[0].num_selected = p['clusters_selected']
if(p.has_key('do_cosinedistance') and p['do_cosinedistance']):
self.net.layer[0].do_cosinedistance = True
print('cosine set to true')
if(p.has_key('zeta_decay') and p['zeta_decay'] is not None):
self.net.layer[0].zeta_matrix = np.ones(self.net.layer[0].weights.shape,dtype=np.float32)
self.net.layer[0].zeta = 1.0
self.zeta_decay = p['zeta_decay']
if(p.has_key('cluster_func2') and p['cluster_func2'] is not None):
print("layer 1 has cluster func")
self.cluster_func2 = p['cluster_func2']
#TODO: Make sure the centroids cover the input space appropriately
self.net.layer[1].centroids = np.asarray(((np.random.random((self.net.layer[1].weights.shape)) - 0.5) * 2.25),np.float32)
#make the centroid bias input match the bias data of 1.0
self.net.layer[1].centroids[:,-1] = 1.0
#print(str(self.net.layer[0].centroids.shape))
#print(str(self.net.layer[0].centroids))
self.net.layer[1].select_func = csf.select_names[p['cluster_func2']]
#print('cluster_func: ' + str(csf.select_names[p['cluster_func']]))
self.net.layer[1].centroid_speed = p['cluster_speed2']
self.net.layer[1].num_selected = p['clusters_selected2']
if(p.has_key('do_cosinedistance') and p['do_cosinedistance']):
self.net.layer[1].do_cosinedistance = True
print('cosine set to true')
if(p.has_key('zeta_decay2') and p['zeta_decay2'] is not None):
self.net.layer[1].zeta_matrix = np.ones(self.net.layer[1].weights.shape,dtype=np.float32)
self.net.layer[1].zeta = 1.0
self.zeta_decay = p['zeta_decay2']
self.do_full_zeta = p.get('do_full_zeta',False)
if(p.has_key('_lambda') and p['_lambda'] is not None):
self._lambda = p['_lambda']
self.gamma = p['gamma']
for l in self.net.layer:
l.eligibility = l.gradient
print("Network Size Check:" )
print(" weight 0 size: " + str(self.net.layer[0].weights.shape))
if(len(self.net.layer) > 1):
print(" weight 1 size: " + str(self.net.layer[1].weights.shape))
if(len(self.net.layer) > 2):
print(" weight 2 size: " + str(self.net.layer[2].weights.shape))
if(len(self.net.layer) > 3):
print(" weight 3 size: " + str(self.net.layer[3].weights.shape))
#(test_data,test_class) = load_data(range(10),"testing",p)
#test_size = test_data.shape[0]
num_hidden = p['num_hidden']
training_epochs = p['training_epochs']
minibatch_size = 128;
#layers = [nnet.layer(28*28),
# nnet.layer(num_hidden,'tanh',select_func=p['select_func'],select_func_params=p['num_selected_neurons']),
# nnet.layer(10,'tanh')]
layers = [];
layers.append(nnet.layer(28*28))
layers.append(nnet.layer(p['num_hidden'],p['activation_function'],select_func=p['select_func'],select_func_params=p['num_selected_neurons']))
#Add 2nd and 3rd hidden layers if there are parameters indicating that we should
if(p.has_key('num_hidden2')):
layers.append(nnet.layer(p['num_hidden2'],p['activation_function2'],select_func=p['select_func2'],select_func_params=p['num_selected_neurons2']))
if(p.has_key('num_hidden3')):
layers.append(nnet.layer(p['num_hidden3'],p['activation_function3'],select_func=p['select_func3'],select_func_params=p['num_selected_neurons3']))
layers.append(nnet.layer(10,p['activation_function_final']))
learning_rate = p['learning_rate']
np.random.seed(p['random_seed']);
#init net
net = nnet.net(layers,learning_rate)
old_sample_targets = np.random.randint(0,2,(num_old_samples,sample_size))
if(p['zerosandones'] == False):
old_sample_data = (old_sample_data-0.5)*2.0
new_sample_data = (new_sample_data-0.5)*2.0
old_sample_targets = (old_sample_targets-0.5)*2.0
new_sample_targets = (new_sample_targets-0.5)*2.0
num_hidden = p['num_hidden']
training_epochs = p['training_epochs']
minibatch_size = p['minibatch_size'];
layers = [];
layers.append(nnet.layer(sample_size))
layers.append(nnet.layer(p['num_hidden'],p['activation_function'],select_func=p['select_func'],
select_func_params=p['num_selected_neurons'],
initialization_scheme=p['initialization_scheme'],
initialization_constant=p['initialization_constant'],
dropout=p['dropout'],sparse_penalty=p['sparse_penalty'],
sparse_target=p['sparse_target']))
#Add 2nd and 3rd hidden layers if there are parameters indicating that we should
if(p.has_key('num_hidden2')):
layers.append(nnet.layer(p['num_hidden2'],p['activation_function2'],select_func=p['select_func2'],
select_func_params=p['num_selected_neurons2'],
initialization_scheme=p['initialization_scheme2'],
initialization_constant=p['initialization_constant2'],
dropout=p['dropout2'],sparse_penalty=p['sparse_penalty2'],
sparse_target=p['sparse_target2']))
#2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
#
#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
#DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
#SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
#THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import time
import numpy as np
#from nnet_toolkit import nnet_cuda as nnet
from nnet_toolkit import nnet
#layers = [nnet_toolkit.layer(2),nnet_toolkit.layer(128,'squash'),nnet_toolkit.layer(1,'squash')];
layers = [
nnet.layer(400),
nnet.layer(128, 'sigmoid'),
nnet.layer(3, 'sigmoid')
]
#training_data = np.array([[0,0,1,1],[0,1,0,1]]);
#training_out = np.array([0,1,1,0]);
training_data = np.random.random((400, 500))
training_out = np.random.random((3, 500))
#net = nnet.net_cuda(layers,step_size=.1);
net = nnet.net(layers, step_size=.1)
net.input = training_data
t = time.time()
nodes_per_group = p['nodes_per_group']
else:
nodes_per_group = None
if(p.has_key('nodes_per_group2')):
nodes_per_group2 = p['nodes_per_group2']
else:
nodes_per_group2 = None
if(p.has_key('nodes_per_group3')):
nodes_per_group3 = p['nodes_per_group3']
else:
nodes_per_group3 = None
layers = [];
layers.append(nnet.layer(reduce_to))
layers.append(nnet.layer(p['num_hidden'],p['activation_function'],nodes_per_group=nodes_per_group,
initialization_scheme=p['initialization_scheme'],
initialization_constant=p['initialization_constant'],
dropout=p['dropout'],sparse_penalty=p['sparse_penalty'],
sparse_target=p['sparse_target'],use_float32=p['use_float32'],
momentum=p['momentum'],maxnorm=p['maxnorm'],step_size=p['learning_rate']))
#Add 2nd and 3rd hidden layers if there are parameters indicating that we should
if(p.has_key('num_hidden2')):
layers.append(nnet.layer(p['num_hidden2'],p['activation_function2'],nodes_per_group=nodes_per_group2,
initialization_scheme=p['initialization_scheme2'],
initialization_constant=p['initialization_constant2'],
dropout=p['dropout2'],sparse_penalty=p['sparse_penalty2'],
sparse_target=p['sparse_target2'],use_float32=p['use_float32'],
momentum=p['momentum2'],maxnorm=p['maxnorm2'],step_size=p['learning_rate2']))
#1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. # #2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. # #THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE #DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR #SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF #THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import time import numpy as np; #from nnet_toolkit import nnet_cuda as nnet from nnet_toolkit import nnet layers = [nnet.layer(2),nnet.layer(128,'sigmoid'),nnet.layer(1,'sigmoid')]; #layers = [nnet_toolkit.layer(2),nnet_toolkit.layer(256,'linear_rectifier'),nnet_toolkit.layer(128,'linear_rectifier'),nnet_toolkit.layer(64,'linear_rectifier'),nnet_toolkit.layer(32,'linear_rectifier'),nnet_toolkit.layer(1,'squash')]; training_data = np.array([[0,0,1,1],[0,1,0,1]]); training_out = np.array([0,1,1,0]); #net = nnet.net_cuda(layers,step_size=.1); net = nnet.net(layers,step_size=.1); net.input = training_data; t = time.time(); for i in range(100000): net.feed_forward(); net.error = net.output - training_out; net.back_propagate(); net.update_weights();
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import time
import numpy as np
import data.mnist as mnist
from nnet_toolkit import nnet
# Load Data
features, labels = mnist.read(range(9), dataset="training")
tfeatures, tlabels = mnist.read(range(9), dataset="testing")
# Initialize Network
layers = [nnet.layer(features.shape[1]), nnet.layer(128, "sigmoid"), nnet.layer(labels.shape[1], "sigmoid")]
net = nnet.net(layers, step_size=0.1)
# Train Network for N epochs
N = 50
mini_batch_size = 1000
t = time.time()
print "Starting Training..."
for epoch in range(N):
# Randomize Features
rix = np.random.permutation(features.shape[0])
features = features[rix]
labels = labels[rix]
net.input = features
def init_network(pnet,input_size):
num_hidden = pnet['num_hidden']
nodes_per_group = pnet['nodes_per_group']
nodes_per_group2 = pnet['nodes_per_group2']
nodes_per_group3 = pnet['nodes_per_group3']
layers = [];
layers.append(nnet.layer(input_size))
layers.append(nnet.layer(pnet['num_hidden'],pnet['activation_function'],nodes_per_group=nodes_per_group,
initialization_scheme=pnet['initialization_scheme'],
initialization_constant=pnet['initialization_constant'],
dropout=pnet['dropout'],sparse_penalty=pnet['sparse_penalty'],
sparse_target=pnet['sparse_target'],use_float32=pnet['use_float32'],
momentum=pnet['momentum'],maxnorm=pnet['maxnorm'],step_size=pnet['learning_rate']))
#Add 2nd and 3rd hidden layers if there are parameters indicating that we should
if(pnet.has_key('num_hidden2') and pnet['num_hidden2'] is not None):
layers.append(nnet.layer(pnet['num_hidden2'],pnet['activation_function2'],nodes_per_group=nodes_per_group2,
initialization_scheme=pnet['initialization_scheme2'],
initialization_constant=pnet['initialization_constant2'],
dropout=pnet['dropout2'],sparse_penalty=pnet['sparse_penalty2'],
sparse_target=pnet['sparse_target2'],use_float32=pnet['use_float32'],
momentum=pnet['momentum2'],maxnorm=pnet['maxnorm2'],step_size=pnet['learning_rate2']))
if(pnet.has_key('num_hidden3') and pnet['num_hidden3'] is not None):
layers.append(nnet.layer(pnet['num_hidden3'],pnet['activation_function3'],nodes_per_group=nodes_per_group3,
initialization_scheme=pnet['initialization_scheme3'],
initialization_constant=pnet['initialization_constant3'],
dropout=pnet['dropout3'],sparse_penalty=pnet['sparse_penalty3'],
sparse_target=pnet['sparse_target3'],use_float32=pnet['use_float32'],
momentum=pnet['momentum3'],maxnorm=pnet['maxnorm3'],step_size=pnet['learning_rate3']))
layers.append(nnet.layer(num_labels,pnet['activation_function_final'],use_float32=pnet['use_float32'],
step_size=pnet['learning_rate_final'],momentum=pnet['momentum_final']))
#init net
net = nnet.net(layers)
if(pnet.has_key('cluster_func') and pnet['cluster_func'] is not None):
#net.layer[0].centroids = np.asarray((((np.random.random((net.layer[0].weights.shape)) - 0.5)*2.0)),np.float32)
net.layer[0].centroids = np.asarray(((np.ones((net.layer[0].weights.shape))*10.0)),np.float32)
net.layer[0].select_func = csf.select_names[pnet['cluster_func']]
print('cluster_func: ' + str(csf.select_names[pnet['cluster_func']]))
net.layer[0].centroid_speed = pnet['cluster_speed']
net.layer[0].num_selected = pnet['clusters_selected']
net.layer[0].number_to_replace = pnet['number_to_replace']
if(p.has_key('do_cosinedistance') and pnet['do_cosinedistance']):
net.layer[0].do_cosinedistance = True
print('cosine set to true')
if(pnet.has_key('num_hidden2') and pnet.has_key('cluster_func2') and pnet['cluster_func2'] is not None):
#net.layer[0].centroids = np.asarray((((np.random.random((net.layer[0].weights.shape)) - 0.5)*2.0)),np.float32)
net.layer[1].centroids = np.asarray(((np.ones((net.layer[1].weights.shape))*10.0)),np.float32)
net.layer[1].select_func = csf.select_names[pnet['cluster_func2']]
print('cluster_func: ' + str(csf.select_names[pnet['cluster_func2']]))
net.layer[1].centroid_speed = pnet['cluster_speed2']
net.layer[1].num_selected = pnet['clusters_selected2']
net.layer[1].number_to_replace = pnet['number_to_replace']
if(p.has_key('do_cosinedistance') and pnet['do_cosinedistance']):
net.layer[1].do_cosinedistance = True
print('cosine set to true')
if(pnet.has_key('num_hidden3') and pnet.has_key('cluster_func3') and pnet['cluster_func3'] is not None):
#net.layer[0].centroids = np.asarray((((np.random.random((net.layer[0].weights.shape)) - 0.5)*2.0)),np.float32)
net.layer[2].centroids = np.asarray(((np.ones((net.layer[2].weights.shape))*10.0)),np.float32)
net.layer[2].select_func = csf.select_names[pnet['cluster_func3']]
print('cluster_func: ' + str(csf.select_names[pnet['cluster_func3']]))
net.layer[2].centroid_speed = pnet['cluster_speed3']
net.layer[2].num_selected = pnet['clusters_selected3']
net.layer[2].number_to_replace = pnet['number_to_replace']
if(p.has_key('do_cosinedistance') and pnet['do_cosinedistance']):
net.layer[2].do_cosinedistance = True
print('cosine set to true')
net.do_clustering = False
if(pnet['cluster_func'] is not None):
net.do_clustering = True
if(pnet['cluster_func2'] is not None):
net.do_clustering = True
if(pnet['cluster_func3'] is not None):
net.do_clustering = True
return net
#DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
#SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
#THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import time
import numpy as np
import data.mnist as mnist
from nnet_toolkit import nnet
# Load Data
features, labels = mnist.read(range(9), dataset='training')
tfeatures, tlabels = mnist.read(range(9), dataset='testing')
# Initialize Network
layers = [
nnet.layer(features.shape[1]),
nnet.layer(128, 'sigmoid'),
nnet.layer(labels.shape[1], 'sigmoid')
]
net = nnet.net(layers, step_size=.1)
# Train Network for N epochs
N = 50
mini_batch_size = 1000
t = time.time()
print "Starting Training..."
for epoch in range(N):
# Randomize Features
rix = np.random.permutation(features.shape[0])
features = features[rix]
labels = labels[rix]
