def _train(self):
global bias_in
hidden_layers = []
bias_layers = []
compressed_data = copy.copy(
self.unsupervised
) # it isn't compressed at this point, but will be later on
compressed_supervised = self.supervised
mid_layers = self.layers[1:-1] # remove the first and last
for i, current in enumerate(mid_layers):
prior = self.layers[i]
# This accesses the layer before the "current" one,
# since the indexing in mid_layers and self.layers is offset by 1
# print "Compressed data at stage {0} {1}".format(i, compressed_data)
""" build the NN with a bottleneck """
bottleneck = FeedForwardNetwork()
in_layer = LinearLayer(prior)
hidden_layer = self.hidden_layer(current)
out_layer = self.hidden_layer(prior)
bottleneck.addInputModule(in_layer)
bottleneck.addModule(hidden_layer)
bottleneck.addOutputModule(out_layer)
in_to_hidden = FullConnection(in_layer, hidden_layer)
hidden_to_out = FullConnection(hidden_layer, out_layer)
bottleneck.addConnection(in_to_hidden)
bottleneck.addConnection(hidden_to_out)
if self.bias:
bias1 = BiasUnit()
bias2 = BiasUnit()
bottleneck.addModule(bias1)
bottleneck.addModule(bias2)
bias_in = FullConnection(bias1, hidden_layer)
bias_hidden = FullConnection(bias2, out_layer)
bottleneck.addConnection(bias_in)
bottleneck.addConnection(bias_hidden)
bottleneck.sortModules()
print("3here network is okay bottleneck"
) # ====================================
""" train the bottleneck """
print "\n...training for layer ", prior, " to ", current
ds = SupervisedDataSet(prior, prior)
print("5here supervised dataset was built"
) # ==============================
print("8.====================compressed_data_size=============")
print compressed_data.__sizeof__()
if self.dropout_on:
noisy_data, originals = self.dropout(compressed_data,
noise=0.2,
bag=1,
debug=False)
print("6here dropout is begin processing and it's okay"
) # ==============================
print "=============noisylen================"
print len(noisy_data) # =====
for i, n in enumerate(noisy_data):
original = originals[i]
ds.addSample(n, original)
print("7.drop out add nosizy sample success"
) # =============================
else:
for d in (compressed_data):
ds.addSample(d, d)
print("4here begin bp bp bp"
) # ============================================
trainer = BackpropTrainer(bottleneck,
dataset=ds,
learningrate=0.001,
momentum=0.05,
verbose=self.verbose,
weightdecay=0.05)
trainer.trainEpochs(self.compression_epochs)
if self.verbose:
print "...data:\n...", compressed_data[0][:10], \
"\nreconstructed to:\n...", bottleneck.activate(compressed_data[0])[:10]
# just used 10dim of 95 dim mfcc
hidden_layers.append(in_to_hidden)
if self.bias: bias_layers.append(bias_in)
""" use the params from the bottleneck to compress the training data """
compressor = FeedForwardNetwork()
compressor.addInputModule(in_layer)
compressor.addOutputModule(
hidden_layer) # use the hidden layer from above
compressor.addConnection(in_to_hidden)
compressor.sortModules()
compressed_data = [compressor.activate(d) for d in compressed_data]
# del compressed_data #del==============================================
compressed_supervised = [
compressor.activate(d) for d in compressed_supervised
]
# del compressed_supervised #del==============================================
self.nn.append(compressor)
""" Train the softmax layer """
print "\n...training for softmax layer "
softmax = FeedForwardNetwork()
in_layer = LinearLayer(self.layers[-2])
out_layer = self.final_layer(self.layers[-1])
softmax.addInputModule(in_layer)
softmax.addOutputModule(out_layer)
in_to_out = FullConnection(in_layer, out_layer)
softmax.addConnection(in_to_out)
if self.bias:
bias = BiasUnit()
softmax.addModule(bias)
bias_in = FullConnection(bias, out_layer)
softmax.addConnection(bias_in)
softmax.sortModules()
# see if it's for classification or regression
if self.final_layer == SoftmaxLayer:
print "...training for a softmax network"
ds = ClassificationDataSet(self.layers[-2], 1)
else:
print "...training for a regression network"
ds = SupervisedDataSet(self.layers[-2], self.layers[-1])
bag = 1
noisy_data, _ = self.dropout(compressed_supervised,
noise=0.5,
bag=bag,
debug=True)
bagged_targets = []
for t in self.targets:
for b in range(bag):
bagged_targets.append(t)
for i, d in enumerate(noisy_data):
target = bagged_targets[i]
ds.addSample(d, target)
# see if it's for classification or regression
if self.final_layer == SoftmaxLayer:
ds._convertToOneOfMany()
trainer = BackpropTrainer(softmax,
dataset=ds,
learningrate=0.001,
momentum=0.05,
verbose=self.verbose,
weightdecay=0.05)
trainer.trainEpochs(self.compression_epochs)
self.nn.append(softmax)
# print "ABOUT TO APPEND"
# print len(in_to_out.params)
hidden_layers.append(in_to_out)
if self.bias:
bias_layers.append(bias_in)
""" Recreate the whole thing """
# print "hidden layers: " + str(hidden_layers)
# print "bias layers: " + str(bias_layers)
# print "len hidden layers: " + str(len(hidden_layers))
# print "len bias layers: " + str(len(bias_layers))
# connect the first two
autoencoder = FeedForwardNetwork()
first_layer = hidden_layers[0].inmod
next_layer = hidden_layers[0].outmod
autoencoder.addInputModule(first_layer)
connection = FullConnection(first_layer, next_layer)
connection.params[:] = hidden_layers[0].params
autoencoder.addConnection(connection)
# decide whether this should be the output layer or not
if self.autoencoding_only and (len(self.layers) <= 3):
# TODO change this to 2 when you aren't using the softmax above
autoencoder.addOutputModule(next_layer)
else:
autoencoder.addModule(next_layer)
if self.bias:
bias = bias_layers[0]
bias_unit = bias.inmod
autoencoder.addModule(bias_unit)
connection = FullConnection(bias_unit, next_layer)
# print bias.params
connection.params[:] = bias.params
autoencoder.addConnection(connection)
# print connection.params
# connect the middle layers
for i, h in enumerate(hidden_layers[1:-1]):
new_next_layer = h.outmod
# decide whether this should be the output layer or not
if self.autoencoding_only and i == (len(hidden_layers) - 3):
autoencoder.addOutputModule(new_next_layer)
else:
autoencoder.addModule(new_next_layer)
connection = FullConnection(next_layer, new_next_layer)
connection.params[:] = h.params
autoencoder.addConnection(connection)
next_layer = new_next_layer
if self.bias:
bias = bias_layers[i + 1]
bias_unit = bias.inmod
autoencoder.addModule(bias_unit)
connection = FullConnection(bias_unit, next_layer)
connection.params[:] = bias.params
autoencoder.addConnection(connection)
return autoencoder, hidden_layers, next_layer, bias_layers
def _train(self):
hidden_layers = []
bias_layers = []
compressed_data = copy.copy(
self.unsupervised
) # it isn't compressed at this point, but will be later on
compressed_supervised = self.supervised
mid_layers = self.layers[1:-1] # remove the first and last
for i, current in enumerate(mid_layers):
prior = self.layers[
i] # This accesses the layer before the "current" one, since the indexing in mid_layers and self.layers is offset by 1
# build the NN with a bottleneck
bottleneck = FeedForwardNetwork()
in_layer = LinearLayer(prior)
hidden_layer = self.hidden_layer(current)
out_layer = self.hidden_layer(prior)
bottleneck.addInputModule(in_layer)
bottleneck.addModule(hidden_layer)
bottleneck.addOutputModule(out_layer)
in_to_hidden = FullConnection(in_layer, hidden_layer)
hidden_to_out = FullConnection(hidden_layer, out_layer)
bottleneck.addConnection(in_to_hidden)
bottleneck.addConnection(hidden_to_out)
if self.bias:
bias1 = BiasUnit()
bias2 = BiasUnit()
bottleneck.addModule(bias1)
bottleneck.addModule(bias2)
bias_in = FullConnection(bias1, hidden_layer)
bias_hidden = FullConnection(bias2, out_layer)
bottleneck.addConnection(bias_in)
bottleneck.addConnection(bias_hidden)
bottleneck.sortModules()
# train the bottleneck
print "\n...training for layer ", prior, " to ", current
ds = SupervisedDataSet(prior, prior)
if self.dropout_on:
noisy_data, originals = self.dropout(compressed_data,
noise=0.2,
bag=1)
for i, n in enumerate(noisy_data):
original = originals[i]
ds.addSample(n, original)
else:
for d in (compressed_data):
ds.addSample(d, d)
trainer = BackpropTrainer(bottleneck,
dataset=ds,
learningrate=0.001,
momentum=0.05,
verbose=self.verbose,
weightdecay=0.05)
trainer.trainEpochs(self.compression_epochs)
if self.verbose:
print "...data:\n...", compressed_data[
0][:8], "\nreconstructed to:\n...", bottleneck.activate(
compressed_data[0])[:8]
hidden_layers.append(in_to_hidden)
if self.bias: bias_layers.append(bias_in)
# use the params from the bottleneck to compress the training data
compressor = FeedForwardNetwork()
compressor.addInputModule(in_layer)
compressor.addOutputModule(
hidden_layer) # use the hidden layer from above
compressor.addConnection(in_to_hidden)
compressor.sortModules()
compressed_data = [compressor.activate(d) for d in compressed_data]
compressed_supervised = [
compressor.activate(d) for d in compressed_supervised
]
self.nn.append(compressor)
# Train the softmax layer
print "\n...training for softmax layer "
softmax = FeedForwardNetwork()
in_layer = LinearLayer(self.layers[-2])
out_layer = self.final_layer(self.layers[-1])
softmax.addInputModule(in_layer)
softmax.addOutputModule(out_layer)
in_to_out = FullConnection(in_layer, out_layer)
softmax.addConnection(in_to_out)
if self.bias:
bias = BiasUnit()
softmax.addModule(bias)
bias_in = FullConnection(bias, out_layer)
softmax.addConnection(bias_in)
softmax.sortModules()
# see if it's for classification or regression
if self.final_layer == SoftmaxLayer:
print "...training for a softmax network"
ds = ClassificationDataSet(self.layers[-2], 1)
else:
print "...training for a regression network"
ds = SupervisedDataSet(self.layers[-2], self.layers[-1])
bag = 1
noisy_data, _ = self.dropout(compressed_supervised, noise=0.5, bag=bag)
bagged_targets = []
for t in self.targets:
for b in range(bag):
bagged_targets.append(t)
for i, d in enumerate(noisy_data):
target = bagged_targets[i]
ds.addSample(d, target)
# see if it's for classification or regression
if self.final_layer == SoftmaxLayer:
ds._convertToOneOfMany()
# TODO make these configurable
trainer = BackpropTrainer(softmax,
dataset=ds,
learningrate=0.001,
momentum=0.05,
verbose=self.verbose,
weightdecay=0.05)
trainer.trainEpochs(self.compression_epochs)
self.nn.append(softmax)
hidden_layers.append(in_to_out)
if self.bias: bias_layers.append(bias_in)
# Recreate the whole thing
# connect the first two
autoencoder = FeedForwardNetwork()
first_layer = hidden_layers[0].inmod
next_layer = hidden_layers[0].outmod
autoencoder.addInputModule(first_layer)
connection = FullConnection(first_layer, next_layer)
connection.params[:] = hidden_layers[0].params
autoencoder.addConnection(connection)
# decide whether this should be the output layer or not
if self.autoencoding_only and (
len(self.layers) <= 3
): # TODO change this to 2 when you aren't using the softmax above
autoencoder.addOutputModule(next_layer)
else:
autoencoder.addModule(next_layer)
if self.bias:
bias = bias_layers[0]
bias_unit = bias.inmod
autoencoder.addModule(bias_unit)
connection = FullConnection(bias_unit, next_layer)
connection.params[:] = bias.params
autoencoder.addConnection(connection)
# connect the middle layers
for i, h in enumerate(hidden_layers[1:-1]):
new_next_layer = h.outmod
# decide whether this should be the output layer or not
if self.autoencoding_only and i == (len(hidden_layers) - 3):
autoencoder.addOutputModule(new_next_layer)
else:
autoencoder.addModule(new_next_layer)
connection = FullConnection(next_layer, new_next_layer)
connection.params[:] = h.params
autoencoder.addConnection(connection)
next_layer = new_next_layer
if self.bias:
bias = bias_layers[i + 1]
bias_unit = bias.inmod
autoencoder.addModule(bias_unit)
connection = FullConnection(bias_unit, next_layer)
connection.params[:] = bias.params
autoencoder.addConnection(connection)
return autoencoder, hidden_layers, next_layer, bias_layers
def _train(self):
global bias_in
hidden_layers = []
bias_layers = []
compressed_data = copy.copy(self.unsupervised) # it isn't compressed at this point, but will be later on
compressed_supervised = self.supervised
mid_layers = self.layers[1:-1] # remove the first and last
for i, current in enumerate(mid_layers):
prior = self.layers[i]
# This accesses the layer before the "current" one,
# since the indexing in mid_layers and self.layers is offset by 1
# print "Compressed data at stage {0} {1}".format(i, compressed_data)
""" build the NN with a bottleneck """
bottleneck = FeedForwardNetwork()
in_layer = LinearLayer(prior)
hidden_layer = self.hidden_layer(current)
out_layer = self.hidden_layer(prior)
bottleneck.addInputModule(in_layer)
bottleneck.addModule(hidden_layer)
bottleneck.addOutputModule(out_layer)
in_to_hidden = FullConnection(in_layer, hidden_layer)
hidden_to_out = FullConnection(hidden_layer, out_layer)
bottleneck.addConnection(in_to_hidden)
bottleneck.addConnection(hidden_to_out)
if self.bias:
bias1 = BiasUnit()
bias2 = BiasUnit()
bottleneck.addModule(bias1)
bottleneck.addModule(bias2)
bias_in = FullConnection(bias1, hidden_layer)
bias_hidden = FullConnection(bias2, out_layer)
bottleneck.addConnection(bias_in)
bottleneck.addConnection(bias_hidden)
bottleneck.sortModules()
print("3here network is okay bottleneck") # ====================================
""" train the bottleneck """
print "\n...training for layer ", prior, " to ", current
ds = SupervisedDataSet(prior, prior)
print ("5here supervised dataset was built") # ==============================
print("8.====================compressed_data_size=============")
print compressed_data.__sizeof__()
if self.dropout_on:
noisy_data, originals = self.dropout(compressed_data, noise=0.2, bag=1, debug=False)
print("6here dropout is begin processing and it's okay") # ==============================
print "=============noisylen================"
print len(noisy_data) # =====
for i, n in enumerate(noisy_data):
original = originals[i]
ds.addSample(n, original)
print("7.drop out add nosizy sample success") # =============================
else:
for d in (compressed_data):
ds.addSample(d, d)
print("4here begin bp bp bp") # ============================================
trainer = BackpropTrainer(bottleneck, dataset=ds, learningrate=0.001, momentum=0.05,
verbose=self.verbose, weightdecay=0.05)
trainer.trainEpochs(self.compression_epochs)
if self.verbose:
print "...data:\n...", compressed_data[0][:10], \
"\nreconstructed to:\n...", bottleneck.activate(compressed_data[0])[:10]
# just used 10dim of 95 dim mfcc
hidden_layers.append(in_to_hidden)
if self.bias: bias_layers.append(bias_in)
""" use the params from the bottleneck to compress the training data """
compressor = FeedForwardNetwork()
compressor.addInputModule(in_layer)
compressor.addOutputModule(hidden_layer) # use the hidden layer from above
compressor.addConnection(in_to_hidden)
compressor.sortModules()
compressed_data = [compressor.activate(d) for d in compressed_data]
# del compressed_data #del==============================================
compressed_supervised = [compressor.activate(d) for d in compressed_supervised]
# del compressed_supervised #del==============================================
self.nn.append(compressor)
""" Train the softmax layer """
print "\n...training for softmax layer "
softmax = FeedForwardNetwork()
in_layer = LinearLayer(self.layers[-2])
out_layer = self.final_layer(self.layers[-1])
softmax.addInputModule(in_layer)
softmax.addOutputModule(out_layer)
in_to_out = FullConnection(in_layer, out_layer)
softmax.addConnection(in_to_out)
if self.bias:
bias = BiasUnit()
softmax.addModule(bias)
bias_in = FullConnection(bias, out_layer)
softmax.addConnection(bias_in)
softmax.sortModules()
# see if it's for classification or regression
if self.final_layer == SoftmaxLayer:
print "...training for a softmax network"
ds = ClassificationDataSet(self.layers[-2], 1)
else:
print "...training for a regression network"
ds = SupervisedDataSet(self.layers[-2], self.layers[-1])
bag = 1
noisy_data, _ = self.dropout(compressed_supervised, noise=0.5, bag=bag, debug=True)
bagged_targets = []
for t in self.targets:
for b in range(bag):
bagged_targets.append(t)
for i, d in enumerate(noisy_data):
target = bagged_targets[i]
ds.addSample(d, target)
# see if it's for classification or regression
if self.final_layer == SoftmaxLayer:
ds._convertToOneOfMany()
trainer = BackpropTrainer(softmax, dataset=ds, learningrate=0.001,
momentum=0.05, verbose=self.verbose, weightdecay=0.05)
trainer.trainEpochs(self.compression_epochs)
self.nn.append(softmax)
# print "ABOUT TO APPEND"
# print len(in_to_out.params)
hidden_layers.append(in_to_out)
if self.bias:
bias_layers.append(bias_in)
""" Recreate the whole thing """
# print "hidden layers: " + str(hidden_layers)
# print "bias layers: " + str(bias_layers)
# print "len hidden layers: " + str(len(hidden_layers))
# print "len bias layers: " + str(len(bias_layers))
# connect the first two
autoencoder = FeedForwardNetwork()
first_layer = hidden_layers[0].inmod
next_layer = hidden_layers[0].outmod
autoencoder.addInputModule(first_layer)
connection = FullConnection(first_layer, next_layer)
connection.params[:] = hidden_layers[0].params
autoencoder.addConnection(connection)
# decide whether this should be the output layer or not
if self.autoencoding_only and (len(self.layers) <= 3):
# TODO change this to 2 when you aren't using the softmax above
autoencoder.addOutputModule(next_layer)
else:
autoencoder.addModule(next_layer)
if self.bias:
bias = bias_layers[0]
bias_unit = bias.inmod
autoencoder.addModule(bias_unit)
connection = FullConnection(bias_unit, next_layer)
# print bias.params
connection.params[:] = bias.params
autoencoder.addConnection(connection)
# print connection.params
# connect the middle layers
for i, h in enumerate(hidden_layers[1:-1]):
new_next_layer = h.outmod
# decide whether this should be the output layer or not
if self.autoencoding_only and i == (len(hidden_layers) - 3):
autoencoder.addOutputModule(new_next_layer)
else:
autoencoder.addModule(new_next_layer)
connection = FullConnection(next_layer, new_next_layer)
connection.params[:] = h.params
autoencoder.addConnection(connection)
next_layer = new_next_layer
if self.bias:
bias = bias_layers[i + 1]
bias_unit = bias.inmod
autoencoder.addModule(bias_unit)
connection = FullConnection(bias_unit, next_layer)
connection.params[:] = bias.params
autoencoder.addConnection(connection)
return autoencoder, hidden_layers, next_layer, bias_layers
def _train(self):
hidden_layers = []
bias_layers = []
compressed_data = copy.copy(self.unsupervised) # it isn't compressed at this point, but will be later on
compressed_supervised = self.supervised
mid_layers = self.layers[1:-1] # remove the first and last
for i,current in enumerate(mid_layers):
prior = self.layers[i] # This accesses the layer before the "current" one, since the indexing in mid_layers and self.layers is offset by 1
# build the NN with a bottleneck
bottleneck = FeedForwardNetwork()
in_layer = LinearLayer(prior)
hidden_layer = self.hidden_layer(current)
out_layer = self.hidden_layer(prior)
bottleneck.addInputModule(in_layer)
bottleneck.addModule(hidden_layer)
bottleneck.addOutputModule(out_layer)
in_to_hidden = FullConnection(in_layer, hidden_layer)
hidden_to_out = FullConnection(hidden_layer, out_layer)
bottleneck.addConnection(in_to_hidden)
bottleneck.addConnection(hidden_to_out)
if self.bias:
bias1 = BiasUnit()
bias2 = BiasUnit()
bottleneck.addModule(bias1)
bottleneck.addModule(bias2)
bias_in = FullConnection(bias1, hidden_layer)
bias_hidden = FullConnection(bias2, out_layer)
bottleneck.addConnection(bias_in)
bottleneck.addConnection(bias_hidden)
bottleneck.sortModules()
# train the bottleneck
print "\n...training for layer ", prior, " to ", current
ds = SupervisedDataSet(prior,prior)
if self.dropout_on:
noisy_data, originals = self.dropout(compressed_data, noise=0.2, bag=1)
for i,n in enumerate(noisy_data):
original = originals[i]
ds.addSample(n, original)
else:
for d in (compressed_data): ds.addSample(d, d)
trainer = BackpropTrainer(bottleneck, dataset=ds, learningrate=0.001, momentum=0.05, verbose=self.verbose, weightdecay=0.05)
trainer.trainEpochs(self.compression_epochs)
if self.verbose: print "...data:\n...", compressed_data[0][:8], "\nreconstructed to:\n...", bottleneck.activate(compressed_data[0])[:8]
hidden_layers.append(in_to_hidden)
if self.bias: bias_layers.append(bias_in)
# use the params from the bottleneck to compress the training data
compressor = FeedForwardNetwork()
compressor.addInputModule(in_layer)
compressor.addOutputModule(hidden_layer) # use the hidden layer from above
compressor.addConnection(in_to_hidden)
compressor.sortModules()
compressed_data = [compressor.activate(d) for d in compressed_data]
compressed_supervised = [compressor.activate(d) for d in compressed_supervised]
self.nn.append(compressor)
# Train the softmax layer
print "\n...training for softmax layer "
softmax = FeedForwardNetwork()
in_layer = LinearLayer(self.layers[-2])
out_layer = self.final_layer(self.layers[-1])
softmax.addInputModule(in_layer)
softmax.addOutputModule(out_layer)
in_to_out = FullConnection(in_layer, out_layer)
softmax.addConnection(in_to_out)
if self.bias:
bias = BiasUnit()
softmax.addModule(bias)
bias_in = FullConnection(bias, out_layer)
softmax.addConnection(bias_in)
softmax.sortModules()
# see if it's for classification or regression
if self.final_layer == SoftmaxLayer:
print "...training for a softmax network"
ds = ClassificationDataSet(self.layers[-2], 1)
else:
print "...training for a regression network"
ds = SupervisedDataSet(self.layers[-2], self.layers[-1])
bag = 1
noisy_data, _ = self.dropout(compressed_supervised, noise=0.5, bag=bag)
bagged_targets = []
for t in self.targets:
for b in range(bag):
bagged_targets.append(t)
for i,d in enumerate(noisy_data):
target = bagged_targets[i]
ds.addSample(d, target)
# see if it's for classification or regression
if self.final_layer == SoftmaxLayer:
ds._convertToOneOfMany()
# TODO make these configurable
trainer = BackpropTrainer(softmax, dataset=ds, learningrate=0.001, momentum=0.05, verbose=self.verbose, weightdecay=0.05)
trainer.trainEpochs(self.compression_epochs)
self.nn.append(softmax)
hidden_layers.append(in_to_out)
if self.bias: bias_layers.append(bias_in)
# Recreate the whole thing
# connect the first two
autoencoder = FeedForwardNetwork()
first_layer = hidden_layers[0].inmod
next_layer = hidden_layers[0].outmod
autoencoder.addInputModule(first_layer)
connection = FullConnection(first_layer, next_layer)
connection.params[:] = hidden_layers[0].params
autoencoder.addConnection(connection)
# decide whether this should be the output layer or not
if self.autoencoding_only and (len(self.layers) <= 3): # TODO change this to 2 when you aren't using the softmax above
autoencoder.addOutputModule(next_layer)
else:
autoencoder.addModule(next_layer)
if self.bias:
bias = bias_layers[0]
bias_unit = bias.inmod
autoencoder.addModule(bias_unit)
connection = FullConnection(bias_unit, next_layer)
connection.params[:] = bias.params
autoencoder.addConnection(connection)
# connect the middle layers
for i,h in enumerate(hidden_layers[1:-1]):
new_next_layer = h.outmod
# decide whether this should be the output layer or not
if self.autoencoding_only and i == (len(hidden_layers) - 3):
autoencoder.addOutputModule(new_next_layer)
else:
autoencoder.addModule(new_next_layer)
connection = FullConnection(next_layer, new_next_layer)
connection.params[:] = h.params
autoencoder.addConnection(connection)
next_layer = new_next_layer
if self.bias:
bias = bias_layers[i+1]
bias_unit = bias.inmod
autoencoder.addModule(bias_unit)
connection = FullConnection(bias_unit, next_layer)
connection.params[:] = bias.params
autoencoder.addConnection(connection)
return autoencoder, hidden_layers, next_layer, bias_layers
