def __init__(self, number_of_vectors, number_of_input_elements,
number_of_output_elements, number_of_neurons,
number_of_hidden_layers, problem_type, function_types, seed):
self.number_of_vectors = number_of_vectors
self.number_of_input_elements = number_of_input_elements
self.number_of_output_elements = number_of_output_elements
self.number_of_neurons = number_of_neurons
self.number_of_hidden_layers = number_of_hidden_layers
self.function_types = function_types
self.problem_type = problem_type
# Генерация входных и выходных данных при задаче классификации
if problem_type == 'classification':
self.input_layer = lrs.InputLayer(number_of_vectors,
number_of_input_elements, seed)
lrs.InputLayer.generate_classification(self.input_layer)
self.output_layer = lrs.OutputLayer(number_of_vectors,
number_of_output_elements)
lrs.OutputLayer.generate_classification(self.output_layer,
self.input_layer)
# Генерация входных и выходных данных при задаче регрессии
else:
self.input_layer = lrs.InputLayer(number_of_vectors,
number_of_input_elements, seed)
lrs.InputLayer.generate_regression(self.input_layer)
self.output_layer = lrs.OutputLayer(number_of_vectors,
number_of_output_elements)
lrs.OutputLayer.generate_regression(self.output_layer,
self.input_layer)
# Список всех слоев, включая слои входных, выходных данных и слои функций активации
self.layers = list()
self.layers.append(self.input_layer)
for i in range(number_of_hidden_layers):
self.layers.append(
lrs.HiddenLayer(number_of_vectors, number_of_input_elements,
number_of_neurons, i))
self.layers.append(
lrs.ActivationFunction(number_of_vectors,
number_of_input_elements,
number_of_neurons, function_types[i]))
# Выходной слой
self.layers.append(self.output_layer)
# Функция потерь
self.layers.append(
lrs.ActivationFunction(number_of_vectors, number_of_input_elements,
number_of_neurons, function_types[-1]))
# Веса выходного слоя генерируются здесь
lrs.OutputLayer.generate_weights(self.output_layer,
self.layers[-4].number_of_neurons)
def set_rec_net(num_filters,filtsize,superres = 1,nonlin = lnl.LeakyRectify(0.2), AR = False, n_rnn_units = 128, n_features = 13):
input_l = ll.InputLayer((None, None))
rec_nn = ll.DimshuffleLayer(input_l, (0, 'x', 1))
hevar = np.sqrt(np.sqrt(2/(1+0.2**2)))
if nonlin == lnl.tanh:
init = lasagne.init.GlorotUniform()
else:
init = lasagne.init.HeNormal(hevar)
for num,size in zip(num_filters,filtsize):
rec_nn = (ll.Conv1DLayer(rec_nn, num_filters = num, filter_size = size, stride =1, pad = 'same',
nonlinearity = nonlin, name='conv1', W = init))
if not AR:
prob_nn = (ll.Conv1DLayer(rec_nn, num_filters = superres,filter_size = 11,stride =1, pad = 'same',
nonlinearity=lnl.sigmoid,name='conv2', b = lasagne.init.Constant(-3.)))
prob_nn = ll.DimshuffleLayer(prob_nn,(0,2,1))
prob_nn = ll.FlattenLayer(prob_nn)
else:
prob_nn = (ll.Conv1DLayer(rec_nn, num_filters = n_features,filter_size = 11,stride =1, pad = 'same',
nonlinearity=nonlin,name='conv2'))
prob_nn = ll.DimshuffleLayer(prob_nn,(0,2,1))
return {'network':prob_nn,'input':input_l, 'superres':superres, 'n_features':n_features, 'rnn_units':n_rnn_units}
def fcl02():
net = n.NeuralNetwork([
l.InputLayer(height=28, width=28),
l.FullyConnectedLayer(
10, init_func=f.glorot_uniform, act_func=f.softmax)
], f.categorical_crossentropy)
optimizer = o.SGD(0.1)
num_epochs = 1
batch_size = 10
return net, optimizer, num_epochs, batch_size
def FCL1():
net = n.NeuralNetwork([
l.InputLayer(height=8, width=8),
l.FCLayer(10, init_val=o.randSc, activator=o.sigmoid)
], o.qu)
optimizer = o.SGD(0.1)
num_epochs = 2000
batch_size = 100
num_class = 10
return net, optimizer, num_epochs, batch_size, num_class
def fcl01():
net = n.NeuralNetwork([
l.InputLayer(height=28, width=28),
l.FullyConnectedLayer(
100, init_func=f.glorot_uniform, act_func=f.sigmoid),
l.FullyConnectedLayer(
10, init_func=f.glorot_uniform, act_func=f.sigmoid)
], f.quadratic)
optimizer = o.SGD(3.0)
num_epochs = 1
batch_size = 100
return net, optimizer, num_epochs, batch_size
def CNN():
net = n.NeuralNetwork([
l.InputLayer(height=28, width=28),
l.ConvLayer(10, kernel=3, init_val=o.randSc, activator=o.sigmoid),
l.MaxLayer(pool=2),
l.FCLayer(height=10, init_val=o.randSc, activator=o.softmax)
], o.crossentropy)
optimizer = o.SGD(0.1)
num_epochs = 2
batch_size = 50
num_class = 2
return net, optimizer, num_epochs, batch_size, num_class
def cnn02():
net = n.NeuralNetwork([
l.InputLayer(height=28, width=28),
l.ConvolutionalLayer(
2, kernel_size=5, init_func=f.glorot_uniform, act_func=f.sigmoid),
l.MaxPoolingLayer(pool_size=3),
l.FullyConnectedLayer(
height=10, init_func=f.glorot_uniform, act_func=f.softmax)
], f.categorical_crossentropy)
optimizer = o.SGD(0.1)
num_epochs = 2
batch_size = 8
return net, optimizer, num_epochs, batch_size
def cnn01():
net = n.NeuralNetwork([
l.InputLayer(height=28, width=28),
l.ConvolutionalLayer(
2, kernel_size=5, init_func=f.glorot_uniform, act_func=f.sigmoid),
l.MaxPoolingLayer(pool_size=2),
l.FullyConnectedLayer(
height=10, init_func=f.glorot_uniform, act_func=f.softmax)
], f.log_likelihood)
optimizer = o.SGD(0.1)
num_epochs = 3
batch_size = 10
return net, optimizer, num_epochs, batch_size
def cnn(weights):
conv = l.ConvolutionalLayer(2, kernel_size=5, init_func=f.zero, act_func=f.sigmoid)
fcl = l.FullyConnectedLayer(height=10, init_func=f.zero, act_func=f.softmax)
net = n.NeuralNetwork([
l.InputLayer(height=28, width=28),
conv,
l.MaxPoolingLayer(pool_size=3),
fcl
], f.categorical_crossentropy)
conv.w = weights["w"][0][0]
conv.b = np.expand_dims(weights["w"][0][1], 1)
fcl.w = np.swapaxes(weights["w"][1][0], 0, 1)
fcl.b = np.expand_dims(weights["w"][1][1], 1)
return net
def setup_architecture(self):
expected_output = []
feature_vector = []
count = 0
for f in os.listdir(self.training_folder):
if count > 0:
break
if '.png' in f:
feature_vector = feature.get_features('train/'+f)
f = f.split('.')
f = f[0]
theline = linecache.getline('trainLabels.csv', int(f) + 1)
theline = theline.strip(' ')
theline = theline.strip('\n')
theline = theline.split(',')
theline = theline[1]
expected_output = self.get_expected_output(theline)
count += 1
self.input_layer = layers.InputLayer(len(feature_vector))
self.get_hidden_layers(self.hidden_layer_sizes)
self.output_layer = layers.OutputLayer(len(expected_output))
if len(self.hidden_layer_sizes) == 0:
self.input_layer.set_architecture(self.output_layer)
else:
self.input_layer.set_architecture(self.hidden_layers[0])
for i, hidden_layer in enumerate(self.hidden_layers):
prevLayer = None
nextLayer = None
if i == 0:
prevLayer = self.input_layer
else:
prevLayer = self.hidden_layers[i-1]
if i == len(self.hidden_layers) - 1:
nextLayer = self.output_layer
else:
nextLayer = self.hidden_layers[i+1]
hidden_layer.set_architecture(prevLayer, nextLayer)
self.output_layer.set_architecture(self.hidden_layers[-1])
def Loaded_nn(name='save(cnn(3_5_2)(0_1_2_8))'):
batch_of_weights = load_weights(name)
ccn_pack, fcl_pack = batch_of_weights
cnn_b, cnn_w = ccn_pack
fcl_b, fcl_w = fcl_pack
cnn_w = map(float, np.delete(cnn_w.split('\n'), -1))
cnn_b = map(float, np.delete(cnn_b.split('\n'), -1))
fcl_w = map(float, np.delete(fcl_w.split(', '), -1))
fcl_b = map(float, np.delete(fcl_b.split(', '), -1))
'''
fcl_w = map(float,np.delete(fcl_w.split('\n'),-1))
fcl_b = map(float,np.delete(fcl_b.split('\n'),-1))
'''
weights = [cnn_w, cnn_b, fcl_w, fcl_b]
con_layer = l.ConvLayer(3,
kernel=5,
init_val=o.randSc,
activator=o.sigmoid)
fc_layer = l.FCLayer(height=10, init_val=o.randSc, activator=o.softmax)
net = NeuralNetwork([
l.InputLayer(height=28, width=28), con_layer,
l.MaxLayer(pool=2), fc_layer
], o.crossentropy)
con_layer.w = np.reshape(weights[0], (np.shape(con_layer.w)))
con_layer.b = np.reshape(weights[1], (np.shape(con_layer.b)))
fc_layer.w = np.reshape(weights[2], (np.shape(fc_layer.w)))
fc_layer.b = np.reshape(weights[3], (np.shape(fc_layer.b)))
return net
def __init__(self, recognition_params, X, S, P, rng, batch_size=1):
super().__init__(recognition_params, X, S, P, rng, batch_size)
ret_dict = {}
self.n_units = recognition_params['rnn_units']
self.n_convfeatures = recognition_params['n_features']
self.init_rnn_forw = theano.shared(
np.zeros([1, self.n_units]).astype(config.floatX))
self.init_rnn_back = theano.shared(
np.zeros([1, self.n_units]).astype(config.floatX))
n_inputs = self.n_convfeatures + 1 + 1 + self.n_units * int(
self.forw_backw)
self.conv_layer = recognition_params['network']
self.conv_output = ll.get_output(self.conv_layer, inputs=self.X)
x_inp = recognition_params['input']
x_inp.input_var = self.X
trace_layer = ll.DimshuffleLayer(x_inp, (0, 1, 'x'))
self.trace_output = ll.get_output(trace_layer)
gru_cell_inp = ll.InputLayer((None, n_inputs))
self.init_rnn_forw_layer = ll.InputLayer(
(None, self.n_units),
input_var=self.init_rnn_forw.repeat(
self.X.shape[0] // self.init_rnn_forw.shape[0], 0))
self.gru_layer_inp = ll.InputLayer((None, None, n_inputs))
self.gru_cell = ll.GRUCell(gru_cell_inp,
self.n_units,
grad_clipping=10.,
name='forw_rnn',
hid_init=self.init_rnn_forw_layer)
self.gru_layer = ll.RecurrentContainerLayer(
{gru_cell_inp: self.gru_layer_inp}, self.gru_cell['output'])
self.p_layer = ll.DenseLayer(
(None, self.n_units + n_inputs - 2),
1,
nonlinearity=lasagne.nonlinearities.sigmoid,
b=lasagne.init.Constant(-5.),
name='dense_output')
hid_0 = self.init_rnn_forw.repeat(
self.X.shape[0] // self.init_rnn_forw.shape[0], 0)
samp_0 = T.zeros([self.X.shape[0], 1])
scan_inp = T.concatenate([self.conv_output, self.trace_output], axis=2)
if self.forw_backw:
inp_back = ll.ConcatLayer([self.conv_layer, trace_layer], axis=2)
init_rnn_back_layer = ll.InputLayer(
(None, self.n_units),
input_var=self.init_rnn_back.repeat(
self.X.shape[0] // self.init_rnn_back.shape[0], 0))
self.back_layer = ll.GRULayer(inp_back,
self.n_units,
backwards=True,
name='back_rnn',
hid_init=init_rnn_back_layer)
self.back_output = ll.get_output(self.back_layer)
scan_inp = T.concatenate([scan_inp, self.back_output], axis=2)
def sample_step(scan_inp_, hid_tm1, samp_tm1):
cell_in = T.concatenate([scan_inp_, samp_tm1], axis=1)
rnn_t_ = self.gru_cell.get_output_for({
'input': cell_in,
'output': hid_tm1
})
prob_in = T.concatenate([cell_in[:, :-2], rnn_t_['output']],
axis=1)
prob_t = self.p_layer.get_output_for(prob_in)
samp_t = srng.binomial(prob_t.shape,
n=1,
p=prob_t,
dtype=theano.config.floatX)
return rnn_t_['output'], samp_t, prob_t
((rnn_t, s_t, p_t), updates) = \
theano.scan(fn=sample_step,
sequences=[scan_inp.dimshuffle(1, 0, 2)], # Scan iterates over first dimension, so we have to put time in front.
outputs_info = [hid_0, samp_0, None, ])
ret_dict['Probs'] = p_t[:, :, 0].T
ret_dict['Spikes'] = s_t[:, :, 0].T
if self.n_genparams:
self.gen_mu_layer = ll.DenseLayer(
(None, None, self.n_units),
self.n_genparams,
nonlinearity=lasagne.nonlinearities.linear,
W=lasagne.init.Normal(std=0.01, mean=0.0),
num_leading_axes=2,
name='dense_gen_mu')
self.gen_sig_layer = ll.DenseLayer(
(None, None, self.n_units),
self.n_genparams,
nonlinearity=lasagne.nonlinearities.softplus,
W=lasagne.init.Normal(std=0.01, mean=-0.0),
b=lasagne.init.Constant(-2.),
num_leading_axes=2,
name='dense_gen_sig')
ret_dict['Gen_mu'] = self.gen_mu_layer.get_output_for(rnn_t).mean(
0)
ret_dict['Gen_sig'] = self.gen_sig_layer.get_output_for(
rnn_t).mean(0)
self.recfunc = theano.function([self.X],
outputs=ret_dict,
updates=updates,
on_unused_input='ignore')
def __init__(self, RecognitionParams, Input, rng, n_samples=1):
'''
h = Q_phi(z|x), where phi are parameters, z is our latent class, and x are data
'''
super().__init__(Input, rng, n_samples)
self.n_units = RecognitionParams['rnn_units']
self.n_convfeatures = RecognitionParams['n_features']
self.conv_back = RecognitionParams['network']
conv_cell = RecognitionParams['network']
conv_cell = ll.DimshuffleLayer(conv_cell, (1, 0, 2))
self.conv_cell = ll.get_output(conv_cell, inputs=self.Input)
inp_cell = RecognitionParams['input']
inp_cell = ll.DimshuffleLayer(inp_cell, (1, 0, 'x'))
self.inp_cell = ll.get_output(inp_cell, inputs=self.Input)
inp_back = RecognitionParams['input']
inp_back = ll.DimshuffleLayer(inp_back, (0, 1, 'x'))
inp_back = ll.ConcatLayer([self.conv_back, inp_back], axis=2)
cell_inp = ll.InputLayer(
(None, self.n_convfeatures + self.n_units + 1 + 1 + 1))
self.cell = rec.GRUCell(cell_inp, self.n_units, grad_clipping=100.)
self.p_out = ll.DenseLayer((None, self.n_units + self.n_convfeatures),
1,
nonlinearity=lasagne.nonlinearities.sigmoid,
b=lasagne.init.Constant(-3.))
hid_0 = T.zeros([self.Input.shape[0], self.n_units])
samp_0 = T.zeros([self.Input.shape[0], 1])
self.back_nn = rec.GRULayer(inp_back, self.n_units, backwards=True)
self.back_nn = ll.DimshuffleLayer(self.back_nn, (1, 0, 2))
self.backward = ll.get_output(self.back_nn, inputs=self.Input)
def sampleStep(conv_cell, inp_cell, back, hid_tm1, samp_tm1, prob_tm1):
cell_in = T.concatenate(
[conv_cell, inp_cell, back, samp_tm1, prob_tm1], axis=1)
rnn_t = self.cell.get_output_for({
'input': cell_in,
'output': hid_tm1
})
prob_in = T.concatenate([conv_cell, rnn_t['output']], axis=1)
prob_t = self.p_out.get_output_for(prob_in)
samp_t = srng.binomial(prob_t.shape,
n=1,
p=prob_t,
dtype=theano.config.floatX)
return rnn_t['output'], samp_t, prob_t
((rnn_temp,s_t, p_t), updates) =\
theano.scan(fn=sampleStep,
sequences=[self.conv_cell,self.inp_cell, self.backward],
# outputs_info=[T.unbroadcast(hid_0,1), T.unbroadcast(samp_0,1), T.unbroadcast(samp_0,1)])
outputs_info=[hid_0, samp_0, samp_0])
for k, v in updates.items():
k.default_update = v
self.recfunc = theano.function([self.Input],
outputs=p_t[:, :, 0].T,
updates=updates)
self.samplefunc = theano.function([self.Input],
outputs=s_t[:, :, 0].T,
updates=updates)
self.dualfunc = theano.function(
[self.Input],
outputs=[p_t[:, :, 0].T, s_t[:, :, 0].T],
updates=updates)
self.detfunc = self.recfunc
def test(self, number_of_vectors):
# Генерация входных и выходных данных при задаче классификации
if self.problem_type == 'classification':
test_input_layer = lrs.InputLayer(number_of_vectors,
self.number_of_input_elements, 2)
lrs.InputLayer.generate_classification(test_input_layer)
test_output_layer = lrs.OutputLayer(number_of_vectors,
self.number_of_output_elements)
lrs.OutputLayer.generate_classification(test_output_layer,
test_input_layer)
# Генерация входных и выходных данных при задаче регрессии
else:
test_input_layer = lrs.InputLayer(number_of_vectors,
self.number_of_input_elements, 2)
lrs.InputLayer.generate_regression(test_input_layer)
test_output_layer = lrs.OutputLayer(number_of_vectors,
self.number_of_output_elements)
lrs.OutputLayer.generate_regression(test_output_layer,
test_input_layer)
test_output_layer.weights = self.output_layer.weights
test_layers = [test_input_layer]
for i in range(len(self.layers) - 3):
test_layers.append(self.layers[i + 1])
test_layers[i + 1].number_of_vectors = number_of_vectors
test_layers.append(test_output_layer)
test_layers.append(
lrs.ActivationFunction(number_of_vectors,
self.number_of_input_elements,
self.number_of_neurons,
self.function_types[-1]))
prediction = self.forward(test_layers)
classes = test_layers[-1].array
print('\nPredicted array (test): ')
print(prediction)
if self.function_types[-1] == 'l2':
squared_error = np.power(
(test_layers[-2].array - test_layers[-2].predicted_array),
2).sum()
mean_squared_error = squared_error / (
number_of_vectors * self.number_of_input_elements)
print('\nMean squared error: ')
print(mean_squared_error)
else:
correct = 0.0
for i in range(number_of_vectors):
prediction_rounded = np.round(classes, 0)
if prediction_rounded[i][0] == test_layers[-2].array[i][0]:
correct += 1
accuracy = correct * 100.0 / number_of_vectors
print('\nAccuracy: %.2f%%' % accuracy)
def set_rec_net(num_filters,
filtsize,
nonlin=lnl.LeakyRectify(0.2),
AR=False,
FB=False,
n_rnn_units=64,
n_features=13,
n_genparams=0,
p_sigma=None):
"""
:param num_filters: Number of filters in each layer for the convolutional network, also sets number of layers
:param filtsize: Size of the filters in each layer
:param nonlin: Nonlinearity used
These parameters are only relevant if a RNN is used to obtain a correlated posterior estimation
:param AR: Whether this network is used as the first stage of an auto-regressive network
:param FB: Whether the auto-regressive network uses a backwards running RNN
:param n_rnn_units: Number of units in the RNN
:param n_features: Number of features passed form the CNN to the RNN
:param n_genparams: Number of generative model parameters inferred by the recognition network
:param p_sigma: standard deviation of the prior on the inffered generative model parameter.
"""
input_l = ll.InputLayer((None, None))
rec_nn = ll.DimshuffleLayer(input_l, (0, 'x', 1))
hevar = np.sqrt(np.sqrt(2 / (1 + 0.2**2)))
convout_nonlin = nonlin
if n_features == 1: convout_nonlin = lnl.linear
if nonlin == lnl.tanh:
init = lasagne.init.GlorotUniform()
else:
init = lasagne.init.HeNormal(hevar)
for num, size in zip(num_filters, filtsize):
rec_nn = (ll.Conv1DLayer(rec_nn,
num_filters=num,
filter_size=size,
stride=1,
pad='same',
nonlinearity=nonlin,
name='conv_filter',
W=init))
if not AR:
prob_nn = (ll.Conv1DLayer(rec_nn,
num_filters=1,
filter_size=11,
stride=1,
pad='same',
nonlinearity=lnl.sigmoid,
name='conv_out',
b=lasagne.init.Constant(-3.)))
prob_nn = ll.DimshuffleLayer(prob_nn, (0, 2, 1))
prob_nn = ll.FlattenLayer(prob_nn)
else:
prob_nn = (ll.Conv1DLayer(rec_nn,
num_filters=n_features,
filter_size=11,
stride=1,
pad='same',
nonlinearity=convout_nonlin,
name='conv_out'))
prob_nn = ll.DimshuffleLayer(prob_nn, (0, 2, 1))
if n_features == 1: prob_nn = ll.FlattenLayer(prob_nn)
return {
'network': prob_nn,
'input': input_l,
'n_features': n_features,
'rnn_units': n_rnn_units,
'n_genparams': n_genparams,
'p_sigma': p_sigma,
'forw_backw': FB
}