def _train(self): global bias_in hidden_layers = [] bias_layers = [] compressed_data = copy.copy( self.unsupervised ) # it isn't compressed at this point, but will be later on compressed_supervised = self.supervised mid_layers = self.layers[1:-1] # remove the first and last for i, current in enumerate(mid_layers): prior = self.layers[i] # This accesses the layer before the "current" one, # since the indexing in mid_layers and self.layers is offset by 1 # print "Compressed data at stage {0} {1}".format(i, compressed_data) """ build the NN with a bottleneck """ bottleneck = FeedForwardNetwork() in_layer = LinearLayer(prior) hidden_layer = self.hidden_layer(current) out_layer = self.hidden_layer(prior) bottleneck.addInputModule(in_layer) bottleneck.addModule(hidden_layer) bottleneck.addOutputModule(out_layer) in_to_hidden = FullConnection(in_layer, hidden_layer) hidden_to_out = FullConnection(hidden_layer, out_layer) bottleneck.addConnection(in_to_hidden) bottleneck.addConnection(hidden_to_out) if self.bias: bias1 = BiasUnit() bias2 = BiasUnit() bottleneck.addModule(bias1) bottleneck.addModule(bias2) bias_in = FullConnection(bias1, hidden_layer) bias_hidden = FullConnection(bias2, out_layer) bottleneck.addConnection(bias_in) bottleneck.addConnection(bias_hidden) bottleneck.sortModules() print("3here network is okay bottleneck" ) # ==================================== """ train the bottleneck """ print "\n...training for layer ", prior, " to ", current ds = SupervisedDataSet(prior, prior) print("5here supervised dataset was built" ) # ============================== print("8.====================compressed_data_size=============") print compressed_data.__sizeof__() if self.dropout_on: noisy_data, originals = self.dropout(compressed_data, noise=0.2, bag=1, debug=False) print("6here dropout is begin processing and it's okay" ) # ============================== print "=============noisylen================" print len(noisy_data) # ===== for i, n in enumerate(noisy_data): original = originals[i] ds.addSample(n, original) print("7.drop out add nosizy sample success" ) # ============================= else: for d in (compressed_data): ds.addSample(d, d) print("4here begin bp bp bp" ) # ============================================ trainer = BackpropTrainer(bottleneck, dataset=ds, learningrate=0.001, momentum=0.05, verbose=self.verbose, weightdecay=0.05) trainer.trainEpochs(self.compression_epochs) if self.verbose: print "...data:\n...", compressed_data[0][:10], \ "\nreconstructed to:\n...", bottleneck.activate(compressed_data[0])[:10] # just used 10dim of 95 dim mfcc hidden_layers.append(in_to_hidden) if self.bias: bias_layers.append(bias_in) """ use the params from the bottleneck to compress the training data """ compressor = FeedForwardNetwork() compressor.addInputModule(in_layer) compressor.addOutputModule( hidden_layer) # use the hidden layer from above compressor.addConnection(in_to_hidden) compressor.sortModules() compressed_data = [compressor.activate(d) for d in compressed_data] # del compressed_data #del============================================== compressed_supervised = [ compressor.activate(d) for d in compressed_supervised ] # del compressed_supervised #del============================================== self.nn.append(compressor) """ Train the softmax layer """ print "\n...training for softmax layer " softmax = FeedForwardNetwork() in_layer = LinearLayer(self.layers[-2]) out_layer = self.final_layer(self.layers[-1]) softmax.addInputModule(in_layer) softmax.addOutputModule(out_layer) in_to_out = FullConnection(in_layer, out_layer) softmax.addConnection(in_to_out) if self.bias: bias = BiasUnit() softmax.addModule(bias) bias_in = FullConnection(bias, out_layer) softmax.addConnection(bias_in) softmax.sortModules() # see if it's for classification or regression if self.final_layer == SoftmaxLayer: print "...training for a softmax network" ds = ClassificationDataSet(self.layers[-2], 1) else: print "...training for a regression network" ds = SupervisedDataSet(self.layers[-2], self.layers[-1]) bag = 1 noisy_data, _ = self.dropout(compressed_supervised, noise=0.5, bag=bag, debug=True) bagged_targets = [] for t in self.targets: for b in range(bag): bagged_targets.append(t) for i, d in enumerate(noisy_data): target = bagged_targets[i] ds.addSample(d, target) # see if it's for classification or regression if self.final_layer == SoftmaxLayer: ds._convertToOneOfMany() trainer = BackpropTrainer(softmax, dataset=ds, learningrate=0.001, momentum=0.05, verbose=self.verbose, weightdecay=0.05) trainer.trainEpochs(self.compression_epochs) self.nn.append(softmax) # print "ABOUT TO APPEND" # print len(in_to_out.params) hidden_layers.append(in_to_out) if self.bias: bias_layers.append(bias_in) """ Recreate the whole thing """ # print "hidden layers: " + str(hidden_layers) # print "bias layers: " + str(bias_layers) # print "len hidden layers: " + str(len(hidden_layers)) # print "len bias layers: " + str(len(bias_layers)) # connect the first two autoencoder = FeedForwardNetwork() first_layer = hidden_layers[0].inmod next_layer = hidden_layers[0].outmod autoencoder.addInputModule(first_layer) connection = FullConnection(first_layer, next_layer) connection.params[:] = hidden_layers[0].params autoencoder.addConnection(connection) # decide whether this should be the output layer or not if self.autoencoding_only and (len(self.layers) <= 3): # TODO change this to 2 when you aren't using the softmax above autoencoder.addOutputModule(next_layer) else: autoencoder.addModule(next_layer) if self.bias: bias = bias_layers[0] bias_unit = bias.inmod autoencoder.addModule(bias_unit) connection = FullConnection(bias_unit, next_layer) # print bias.params connection.params[:] = bias.params autoencoder.addConnection(connection) # print connection.params # connect the middle layers for i, h in enumerate(hidden_layers[1:-1]): new_next_layer = h.outmod # decide whether this should be the output layer or not if self.autoencoding_only and i == (len(hidden_layers) - 3): autoencoder.addOutputModule(new_next_layer) else: autoencoder.addModule(new_next_layer) connection = FullConnection(next_layer, new_next_layer) connection.params[:] = h.params autoencoder.addConnection(connection) next_layer = new_next_layer if self.bias: bias = bias_layers[i + 1] bias_unit = bias.inmod autoencoder.addModule(bias_unit) connection = FullConnection(bias_unit, next_layer) connection.params[:] = bias.params autoencoder.addConnection(connection) return autoencoder, hidden_layers, next_layer, bias_layers
def _train(self): hidden_layers = [] bias_layers = [] compressed_data = copy.copy( self.unsupervised ) # it isn't compressed at this point, but will be later on compressed_supervised = self.supervised mid_layers = self.layers[1:-1] # remove the first and last for i, current in enumerate(mid_layers): prior = self.layers[ i] # This accesses the layer before the "current" one, since the indexing in mid_layers and self.layers is offset by 1 # build the NN with a bottleneck bottleneck = FeedForwardNetwork() in_layer = LinearLayer(prior) hidden_layer = self.hidden_layer(current) out_layer = self.hidden_layer(prior) bottleneck.addInputModule(in_layer) bottleneck.addModule(hidden_layer) bottleneck.addOutputModule(out_layer) in_to_hidden = FullConnection(in_layer, hidden_layer) hidden_to_out = FullConnection(hidden_layer, out_layer) bottleneck.addConnection(in_to_hidden) bottleneck.addConnection(hidden_to_out) if self.bias: bias1 = BiasUnit() bias2 = BiasUnit() bottleneck.addModule(bias1) bottleneck.addModule(bias2) bias_in = FullConnection(bias1, hidden_layer) bias_hidden = FullConnection(bias2, out_layer) bottleneck.addConnection(bias_in) bottleneck.addConnection(bias_hidden) bottleneck.sortModules() # train the bottleneck print "\n...training for layer ", prior, " to ", current ds = SupervisedDataSet(prior, prior) if self.dropout_on: noisy_data, originals = self.dropout(compressed_data, noise=0.2, bag=1) for i, n in enumerate(noisy_data): original = originals[i] ds.addSample(n, original) else: for d in (compressed_data): ds.addSample(d, d) trainer = BackpropTrainer(bottleneck, dataset=ds, learningrate=0.001, momentum=0.05, verbose=self.verbose, weightdecay=0.05) trainer.trainEpochs(self.compression_epochs) if self.verbose: print "...data:\n...", compressed_data[ 0][:8], "\nreconstructed to:\n...", bottleneck.activate( compressed_data[0])[:8] hidden_layers.append(in_to_hidden) if self.bias: bias_layers.append(bias_in) # use the params from the bottleneck to compress the training data compressor = FeedForwardNetwork() compressor.addInputModule(in_layer) compressor.addOutputModule( hidden_layer) # use the hidden layer from above compressor.addConnection(in_to_hidden) compressor.sortModules() compressed_data = [compressor.activate(d) for d in compressed_data] compressed_supervised = [ compressor.activate(d) for d in compressed_supervised ] self.nn.append(compressor) # Train the softmax layer print "\n...training for softmax layer " softmax = FeedForwardNetwork() in_layer = LinearLayer(self.layers[-2]) out_layer = self.final_layer(self.layers[-1]) softmax.addInputModule(in_layer) softmax.addOutputModule(out_layer) in_to_out = FullConnection(in_layer, out_layer) softmax.addConnection(in_to_out) if self.bias: bias = BiasUnit() softmax.addModule(bias) bias_in = FullConnection(bias, out_layer) softmax.addConnection(bias_in) softmax.sortModules() # see if it's for classification or regression if self.final_layer == SoftmaxLayer: print "...training for a softmax network" ds = ClassificationDataSet(self.layers[-2], 1) else: print "...training for a regression network" ds = SupervisedDataSet(self.layers[-2], self.layers[-1]) bag = 1 noisy_data, _ = self.dropout(compressed_supervised, noise=0.5, bag=bag) bagged_targets = [] for t in self.targets: for b in range(bag): bagged_targets.append(t) for i, d in enumerate(noisy_data): target = bagged_targets[i] ds.addSample(d, target) # see if it's for classification or regression if self.final_layer == SoftmaxLayer: ds._convertToOneOfMany() # TODO make these configurable trainer = BackpropTrainer(softmax, dataset=ds, learningrate=0.001, momentum=0.05, verbose=self.verbose, weightdecay=0.05) trainer.trainEpochs(self.compression_epochs) self.nn.append(softmax) hidden_layers.append(in_to_out) if self.bias: bias_layers.append(bias_in) # Recreate the whole thing # connect the first two autoencoder = FeedForwardNetwork() first_layer = hidden_layers[0].inmod next_layer = hidden_layers[0].outmod autoencoder.addInputModule(first_layer) connection = FullConnection(first_layer, next_layer) connection.params[:] = hidden_layers[0].params autoencoder.addConnection(connection) # decide whether this should be the output layer or not if self.autoencoding_only and ( len(self.layers) <= 3 ): # TODO change this to 2 when you aren't using the softmax above autoencoder.addOutputModule(next_layer) else: autoencoder.addModule(next_layer) if self.bias: bias = bias_layers[0] bias_unit = bias.inmod autoencoder.addModule(bias_unit) connection = FullConnection(bias_unit, next_layer) connection.params[:] = bias.params autoencoder.addConnection(connection) # connect the middle layers for i, h in enumerate(hidden_layers[1:-1]): new_next_layer = h.outmod # decide whether this should be the output layer or not if self.autoencoding_only and i == (len(hidden_layers) - 3): autoencoder.addOutputModule(new_next_layer) else: autoencoder.addModule(new_next_layer) connection = FullConnection(next_layer, new_next_layer) connection.params[:] = h.params autoencoder.addConnection(connection) next_layer = new_next_layer if self.bias: bias = bias_layers[i + 1] bias_unit = bias.inmod autoencoder.addModule(bias_unit) connection = FullConnection(bias_unit, next_layer) connection.params[:] = bias.params autoencoder.addConnection(connection) return autoencoder, hidden_layers, next_layer, bias_layers
def _train(self): global bias_in hidden_layers = [] bias_layers = [] compressed_data = copy.copy(self.unsupervised) # it isn't compressed at this point, but will be later on compressed_supervised = self.supervised mid_layers = self.layers[1:-1] # remove the first and last for i, current in enumerate(mid_layers): prior = self.layers[i] # This accesses the layer before the "current" one, # since the indexing in mid_layers and self.layers is offset by 1 # print "Compressed data at stage {0} {1}".format(i, compressed_data) """ build the NN with a bottleneck """ bottleneck = FeedForwardNetwork() in_layer = LinearLayer(prior) hidden_layer = self.hidden_layer(current) out_layer = self.hidden_layer(prior) bottleneck.addInputModule(in_layer) bottleneck.addModule(hidden_layer) bottleneck.addOutputModule(out_layer) in_to_hidden = FullConnection(in_layer, hidden_layer) hidden_to_out = FullConnection(hidden_layer, out_layer) bottleneck.addConnection(in_to_hidden) bottleneck.addConnection(hidden_to_out) if self.bias: bias1 = BiasUnit() bias2 = BiasUnit() bottleneck.addModule(bias1) bottleneck.addModule(bias2) bias_in = FullConnection(bias1, hidden_layer) bias_hidden = FullConnection(bias2, out_layer) bottleneck.addConnection(bias_in) bottleneck.addConnection(bias_hidden) bottleneck.sortModules() print("3here network is okay bottleneck") # ==================================== """ train the bottleneck """ print "\n...training for layer ", prior, " to ", current ds = SupervisedDataSet(prior, prior) print ("5here supervised dataset was built") # ============================== print("8.====================compressed_data_size=============") print compressed_data.__sizeof__() if self.dropout_on: noisy_data, originals = self.dropout(compressed_data, noise=0.2, bag=1, debug=False) print("6here dropout is begin processing and it's okay") # ============================== print "=============noisylen================" print len(noisy_data) # ===== for i, n in enumerate(noisy_data): original = originals[i] ds.addSample(n, original) print("7.drop out add nosizy sample success") # ============================= else: for d in (compressed_data): ds.addSample(d, d) print("4here begin bp bp bp") # ============================================ trainer = BackpropTrainer(bottleneck, dataset=ds, learningrate=0.001, momentum=0.05, verbose=self.verbose, weightdecay=0.05) trainer.trainEpochs(self.compression_epochs) if self.verbose: print "...data:\n...", compressed_data[0][:10], \ "\nreconstructed to:\n...", bottleneck.activate(compressed_data[0])[:10] # just used 10dim of 95 dim mfcc hidden_layers.append(in_to_hidden) if self.bias: bias_layers.append(bias_in) """ use the params from the bottleneck to compress the training data """ compressor = FeedForwardNetwork() compressor.addInputModule(in_layer) compressor.addOutputModule(hidden_layer) # use the hidden layer from above compressor.addConnection(in_to_hidden) compressor.sortModules() compressed_data = [compressor.activate(d) for d in compressed_data] # del compressed_data #del============================================== compressed_supervised = [compressor.activate(d) for d in compressed_supervised] # del compressed_supervised #del============================================== self.nn.append(compressor) """ Train the softmax layer """ print "\n...training for softmax layer " softmax = FeedForwardNetwork() in_layer = LinearLayer(self.layers[-2]) out_layer = self.final_layer(self.layers[-1]) softmax.addInputModule(in_layer) softmax.addOutputModule(out_layer) in_to_out = FullConnection(in_layer, out_layer) softmax.addConnection(in_to_out) if self.bias: bias = BiasUnit() softmax.addModule(bias) bias_in = FullConnection(bias, out_layer) softmax.addConnection(bias_in) softmax.sortModules() # see if it's for classification or regression if self.final_layer == SoftmaxLayer: print "...training for a softmax network" ds = ClassificationDataSet(self.layers[-2], 1) else: print "...training for a regression network" ds = SupervisedDataSet(self.layers[-2], self.layers[-1]) bag = 1 noisy_data, _ = self.dropout(compressed_supervised, noise=0.5, bag=bag, debug=True) bagged_targets = [] for t in self.targets: for b in range(bag): bagged_targets.append(t) for i, d in enumerate(noisy_data): target = bagged_targets[i] ds.addSample(d, target) # see if it's for classification or regression if self.final_layer == SoftmaxLayer: ds._convertToOneOfMany() trainer = BackpropTrainer(softmax, dataset=ds, learningrate=0.001, momentum=0.05, verbose=self.verbose, weightdecay=0.05) trainer.trainEpochs(self.compression_epochs) self.nn.append(softmax) # print "ABOUT TO APPEND" # print len(in_to_out.params) hidden_layers.append(in_to_out) if self.bias: bias_layers.append(bias_in) """ Recreate the whole thing """ # print "hidden layers: " + str(hidden_layers) # print "bias layers: " + str(bias_layers) # print "len hidden layers: " + str(len(hidden_layers)) # print "len bias layers: " + str(len(bias_layers)) # connect the first two autoencoder = FeedForwardNetwork() first_layer = hidden_layers[0].inmod next_layer = hidden_layers[0].outmod autoencoder.addInputModule(first_layer) connection = FullConnection(first_layer, next_layer) connection.params[:] = hidden_layers[0].params autoencoder.addConnection(connection) # decide whether this should be the output layer or not if self.autoencoding_only and (len(self.layers) <= 3): # TODO change this to 2 when you aren't using the softmax above autoencoder.addOutputModule(next_layer) else: autoencoder.addModule(next_layer) if self.bias: bias = bias_layers[0] bias_unit = bias.inmod autoencoder.addModule(bias_unit) connection = FullConnection(bias_unit, next_layer) # print bias.params connection.params[:] = bias.params autoencoder.addConnection(connection) # print connection.params # connect the middle layers for i, h in enumerate(hidden_layers[1:-1]): new_next_layer = h.outmod # decide whether this should be the output layer or not if self.autoencoding_only and i == (len(hidden_layers) - 3): autoencoder.addOutputModule(new_next_layer) else: autoencoder.addModule(new_next_layer) connection = FullConnection(next_layer, new_next_layer) connection.params[:] = h.params autoencoder.addConnection(connection) next_layer = new_next_layer if self.bias: bias = bias_layers[i + 1] bias_unit = bias.inmod autoencoder.addModule(bias_unit) connection = FullConnection(bias_unit, next_layer) connection.params[:] = bias.params autoencoder.addConnection(connection) return autoencoder, hidden_layers, next_layer, bias_layers
def _train(self): hidden_layers = [] bias_layers = [] compressed_data = copy.copy(self.unsupervised) # it isn't compressed at this point, but will be later on compressed_supervised = self.supervised mid_layers = self.layers[1:-1] # remove the first and last for i,current in enumerate(mid_layers): prior = self.layers[i] # This accesses the layer before the "current" one, since the indexing in mid_layers and self.layers is offset by 1 # build the NN with a bottleneck bottleneck = FeedForwardNetwork() in_layer = LinearLayer(prior) hidden_layer = self.hidden_layer(current) out_layer = self.hidden_layer(prior) bottleneck.addInputModule(in_layer) bottleneck.addModule(hidden_layer) bottleneck.addOutputModule(out_layer) in_to_hidden = FullConnection(in_layer, hidden_layer) hidden_to_out = FullConnection(hidden_layer, out_layer) bottleneck.addConnection(in_to_hidden) bottleneck.addConnection(hidden_to_out) if self.bias: bias1 = BiasUnit() bias2 = BiasUnit() bottleneck.addModule(bias1) bottleneck.addModule(bias2) bias_in = FullConnection(bias1, hidden_layer) bias_hidden = FullConnection(bias2, out_layer) bottleneck.addConnection(bias_in) bottleneck.addConnection(bias_hidden) bottleneck.sortModules() # train the bottleneck print "\n...training for layer ", prior, " to ", current ds = SupervisedDataSet(prior,prior) if self.dropout_on: noisy_data, originals = self.dropout(compressed_data, noise=0.2, bag=1) for i,n in enumerate(noisy_data): original = originals[i] ds.addSample(n, original) else: for d in (compressed_data): ds.addSample(d, d) trainer = BackpropTrainer(bottleneck, dataset=ds, learningrate=0.001, momentum=0.05, verbose=self.verbose, weightdecay=0.05) trainer.trainEpochs(self.compression_epochs) if self.verbose: print "...data:\n...", compressed_data[0][:8], "\nreconstructed to:\n...", bottleneck.activate(compressed_data[0])[:8] hidden_layers.append(in_to_hidden) if self.bias: bias_layers.append(bias_in) # use the params from the bottleneck to compress the training data compressor = FeedForwardNetwork() compressor.addInputModule(in_layer) compressor.addOutputModule(hidden_layer) # use the hidden layer from above compressor.addConnection(in_to_hidden) compressor.sortModules() compressed_data = [compressor.activate(d) for d in compressed_data] compressed_supervised = [compressor.activate(d) for d in compressed_supervised] self.nn.append(compressor) # Train the softmax layer print "\n...training for softmax layer " softmax = FeedForwardNetwork() in_layer = LinearLayer(self.layers[-2]) out_layer = self.final_layer(self.layers[-1]) softmax.addInputModule(in_layer) softmax.addOutputModule(out_layer) in_to_out = FullConnection(in_layer, out_layer) softmax.addConnection(in_to_out) if self.bias: bias = BiasUnit() softmax.addModule(bias) bias_in = FullConnection(bias, out_layer) softmax.addConnection(bias_in) softmax.sortModules() # see if it's for classification or regression if self.final_layer == SoftmaxLayer: print "...training for a softmax network" ds = ClassificationDataSet(self.layers[-2], 1) else: print "...training for a regression network" ds = SupervisedDataSet(self.layers[-2], self.layers[-1]) bag = 1 noisy_data, _ = self.dropout(compressed_supervised, noise=0.5, bag=bag) bagged_targets = [] for t in self.targets: for b in range(bag): bagged_targets.append(t) for i,d in enumerate(noisy_data): target = bagged_targets[i] ds.addSample(d, target) # see if it's for classification or regression if self.final_layer == SoftmaxLayer: ds._convertToOneOfMany() # TODO make these configurable trainer = BackpropTrainer(softmax, dataset=ds, learningrate=0.001, momentum=0.05, verbose=self.verbose, weightdecay=0.05) trainer.trainEpochs(self.compression_epochs) self.nn.append(softmax) hidden_layers.append(in_to_out) if self.bias: bias_layers.append(bias_in) # Recreate the whole thing # connect the first two autoencoder = FeedForwardNetwork() first_layer = hidden_layers[0].inmod next_layer = hidden_layers[0].outmod autoencoder.addInputModule(first_layer) connection = FullConnection(first_layer, next_layer) connection.params[:] = hidden_layers[0].params autoencoder.addConnection(connection) # decide whether this should be the output layer or not if self.autoencoding_only and (len(self.layers) <= 3): # TODO change this to 2 when you aren't using the softmax above autoencoder.addOutputModule(next_layer) else: autoencoder.addModule(next_layer) if self.bias: bias = bias_layers[0] bias_unit = bias.inmod autoencoder.addModule(bias_unit) connection = FullConnection(bias_unit, next_layer) connection.params[:] = bias.params autoencoder.addConnection(connection) # connect the middle layers for i,h in enumerate(hidden_layers[1:-1]): new_next_layer = h.outmod # decide whether this should be the output layer or not if self.autoencoding_only and i == (len(hidden_layers) - 3): autoencoder.addOutputModule(new_next_layer) else: autoencoder.addModule(new_next_layer) connection = FullConnection(next_layer, new_next_layer) connection.params[:] = h.params autoencoder.addConnection(connection) next_layer = new_next_layer if self.bias: bias = bias_layers[i+1] bias_unit = bias.inmod autoencoder.addModule(bias_unit) connection = FullConnection(bias_unit, next_layer) connection.params[:] = bias.params autoencoder.addConnection(connection) return autoencoder, hidden_layers, next_layer, bias_layers