def network(dataset, input_list): num_words = len(input_list) #dividing the dataset into training and testing data tstdata, trndata = dataset.splitWithProportion(0.25) #building the network net = RecurrentNetwork() input_layer1 = LinearLayer(num_words, name='input_layer1') input_layer2 = LinearLayer(num_words, name='input_layer2') hidden_layer = TanhLayer(num_words, name='hidden_layer') output_layer = SoftmaxLayer(num_words, name='output_layer') net.addInputModule(input_layer1) net.addInputModule(input_layer2) net.addModule(hidden_layer) net.addOutputModule(output_layer) net.addConnection(FullConnection(input_layer1, hidden_layer, name='in1_to_hidden')) net.addConnection(FullConnection(input_layer2, hidden_layer, name='in2_to_hidden')) net.addConnection(FullConnection(hidden_layer, output_layer, name='hidden_to_output')) net.addConnection(FullConnection(input_layer1, output_layer, name='in1_to_out')) net.addConnection(FullConnection(input_layer2, output_layer, name='in2_to_out')) net.sortModules() #backpropagation trainer = BackpropTrainer(net, dataset=trndata, momentum=0.1, verbose=True, weightdecay=0.01) #error checking part for i in range(10): trainer.trainEpochs(1) trnresult = percentError(trainer.testOnClassData(), trndata['target']) tstresult = percentError(trainer.testOnClassData(dataset=tstdata), tstdata['target']) print "epoch: %4d" % trainer.totalepochs print " train error: %5.10f%%" % trnresult print " test error: %5.10f%%" % tstresult return net
def training_and_testing(): nn = init_neural_network() training = learning.get_labeled_data( '%strain-images-idx3-ubyte.gz' % (database_folder), '%strain-labels-idx1-ubyte.gz' % (database_folder), '%strainig' % (database_folder)) test = learning.get_labeled_data( '%st10k-images-idx3-ubyte.gz' % (database_folder), '%st10k-labels-idx1-ubyte.gz' % (database_folder), '%stest' % (database_folder)) FEATURES = N_INPUT_LAYER print("Caracteristicas a analizar: %i" % FEATURES) testdata = ClassificationDataSet(FEATURES, 1, nb_classes=OUTPUT_LAYER) trainingdata = ClassificationDataSet(FEATURES, 1, nb_classes=OUTPUT_LAYER) for i in range(len(test['data'])): testdata.addSample(test['data'][i], test['label'][i]) for j in range(len(training['data'])): trainingdata.addSample(training['data'][j], training['label'][j]) trainingdata._convertToOneOfMany() testdata._convertToOneOfMany() trainer = BackpropTrainer(nn, dataset=trainingdata, momentum=MOMENTUM, verbose=True, weightdecay=W_DECAY, learningrate=L_RATE, lrdecay=L_DECAY) for i in range(EPOCHS): trainer.trainEpochs(1) trnresult = percentError(trainer.testOnClassData(), trainingdata['class']) tstresult = percentError(trainer.testOnClassData(dataset=testdata), testdata['class']) print("epoch: %4d" % trainer.totalepochs, " train error: %5.2f%%" % trnresult, " test error: %5.2f%%" % tstresult) return nn
def network(dataset, input_list): num_words = len(input_list) #dividing the dataset into training and testing data tstdata, trndata = dataset.splitWithProportion(0.25) #building the network net = RecurrentNetwork() input_layer1 = LinearLayer(num_words, name='input_layer1') input_layer2 = LinearLayer(num_words, name='input_layer2') hidden_layer = TanhLayer(num_words, name='hidden_layer') output_layer = SoftmaxLayer(num_words, name='output_layer') net.addInputModule(input_layer1) net.addInputModule(input_layer2) net.addModule(hidden_layer) net.addOutputModule(output_layer) net.addConnection( FullConnection(input_layer1, hidden_layer, name='in1_to_hidden')) net.addConnection( FullConnection(input_layer2, hidden_layer, name='in2_to_hidden')) net.addConnection( FullConnection(hidden_layer, output_layer, name='hidden_to_output')) net.addConnection( FullConnection(input_layer1, output_layer, name='in1_to_out')) net.addConnection( FullConnection(input_layer2, output_layer, name='in2_to_out')) net.sortModules() #backpropagation trainer = BackpropTrainer(net, dataset=trndata, momentum=0.1, verbose=True, weightdecay=0.01) #error checking part for i in range(10): trainer.trainEpochs(1) trnresult = percentError(trainer.testOnClassData(), trndata['target']) tstresult = percentError(trainer.testOnClassData(dataset=tstdata), tstdata['target']) print "epoch: %4d" % trainer.totalepochs print " train error: %5.10f%%" % trnresult print " test error: %5.10f%%" % tstresult return net
def training_and_testing(): nn= init_neural_network() training = learning.get_labeled_data('%strain-images-idx3-ubyte.gz'%(database_folder), '%strain-labels-idx1-ubyte.gz'%(database_folder) ,'%strainig'%(database_folder)) test = learning.get_labeled_data('%st10k-images-idx3-ubyte.gz'%(database_folder), '%st10k-labels-idx1-ubyte.gz'%(database_folder), '%stest'%(database_folder)) FEATURES = N_INPUT_LAYER print("Caracteristicas a analizar: %i"%FEATURES) testdata = ClassificationDataSet(FEATURES,1,nb_classes=OUTPUT_LAYER) trainingdata = ClassificationDataSet(FEATURES,1,nb_classes=OUTPUT_LAYER) for i in range(len(test['data'])): testdata.addSample(test['data'][i],test['label'][i]) for j in range(len(training['data'])): trainingdata.addSample(training['data'][j],training['label'][j]) trainingdata._convertToOneOfMany() testdata._convertToOneOfMany() trainer = BackpropTrainer(nn,dataset=trainingdata,momentum=MOMENTUM,verbose=True, weightdecay=W_DECAY,learningrate=L_RATE,lrdecay=L_DECAY) for i in range(EPOCHS): trainer.trainEpochs(1) trnresult = percentError(trainer.testOnClassData(), trainingdata['class']) tstresult = percentError(trainer.testOnClassData( dataset=testdata), testdata['class']) print("epoch: %4d" % trainer.totalepochs, " train error: %5.2f%%" % trnresult, " test error: %5.2f%%" % tstresult) return nn
from pybrain.tools.shortcuts import buildNetwork from pybrain.structure import SoftmaxLayer # 输入数据是 19维,输出是两维,隐层设置为5层 # 输出层使用Softmax激活,其他:学习率(learningrate=0.01),学习率衰减(lrdecay=1.0,每次训练一步学习率乘以), # 详细(verbose=False)动量因子(momentum=0最后时步的梯度?),权值衰减?(weightdecay=0.0) n_h = 5 net = buildNetwork(19, n_h, 2, outclass=SoftmaxLayer) # Step 2 : 构建前馈网络标准BP算法 from pybrain.supervised import BackpropTrainer trainer_sd = BackpropTrainer(net, traindata) # # 或者使用累积BP算法,训练次数50次 # trainer_ac = BackpropTrainer(net, traindata, batchlearning=True) # trainer_ac.trainEpochs(50) # err_train, err_valid = trainer_ac.trainUntilConvergence(maxEpochs=50) for i in range(50): # 训练50次,每及测试结果次打印训练结果 trainer_sd.trainEpochs(1) # 训练网络一次, # 引入训练误差和测试误差 from pybrain.utilities import percentError trainresult = percentError(trainer_sd.testOnClassData(), traindata['class']) testresult = percentError(trainer_sd.testOnClassData(dataset=testdata), testdata['class']) # 打印错误率 print('Epoch: %d', trainer_sd.totalepochs, 'train error: ', trainresult, 'test error: ', testresult)
class Classifier: def __init__(self,Xtrain,Ytrain): self._Xtrain=Xtrain self._Ytrain=Ytrain self.features=Xtrain.keys() def Train(self,feat_list=None,type='logreg',gamma=0.0,domeanstd=True,special_bias=None,add_bias=True, weight=None, class_instance=None, method='sigmoid',factor=10.0,arch=[10], cv_feats=None, cv_special_bias=None,cv_class_instance=None): if feat_list==None: feat_list=self.features self.feat_list=feat_list self._gamma=gamma self._type=type self._special_bias = special_bias self._add_bias = add_bias Xtrain_feats = np.ascontiguousarray(np.hstack((self._Xtrain[feat] for feat in feat_list))) self.m, self.std = classifier.feature_meanstd(Xtrain_feats) if domeanstd==False: #hacky, overwrite the things we computed self.m[:] = 0 self.std[:] = 1 Xtrain_feats -= self.m Xtrain_feats /= self.std if special_bias != None: Xtrain_feats = np.ascontiguousarray(np.hstack((Xtrain_feats, special_bias))) #CV if cv_feats!=None: cv_feats = np.ascontiguousarray(np.hstack((cv_feats[feat] for feat in feat_list))) cv_feats -= self.m cv_feats /= self.std if special_bias != None: cv_feats = np.ascontiguousarray(np.hstack((cv_feats, cv_special_bias))) '''Classifier stage''' if type=='linsvm': self.w, self.b = classifier.svm_onevsall(Xtrain_feats, self._Ytrain, self._gamma, weight = weight, special_bias=special_bias, add_bias=add_bias) return (self.w,self.b) elif type=='logreg': self.w, self.b = l2logreg_onevsall(Xtrain_feats, self._Ytrain, self._gamma, weight = weight, special_bias=special_bias, add_bias=add_bias) return (self.w,self.b) elif type=='logreg_atwv': self.w, self.b = Train_atwv(Xtrain_feats,class_instance=class_instance,weight=weight,special_bias=special_bias, add_bias=add_bias, method=method, factor=factor, gamma=self._gamma, cv_class_instance=cv_class_instance, cv_feats=cv_feats) elif type=='nn_atwv': self._arch = arch self._weights_nn = Train_atwv_nn(Xtrain_feats,class_instance=class_instance,weight=weight,special_bias=special_bias, add_bias=add_bias, arch=self._arch, method=method, factor=factor, gamma=self._gamma, cv_class_instance=cv_class_instance, cv_feats=cv_feats) #self._weights_nn = Train_atwv_nn(Xtrain_feats,class_instance=class_instance,weight=self._weights_nn,special_bias=special_bias, add_bias=add_bias, # arch=self._arch, method=method, factor=factor*10.0) elif type=='nn_debug': if mpi.COMM.Get_size() > 1: print 'Warning!!! Running NN training with MPI with more than one Node!' #FIXME: Collect X and Y at root to avoid this # prob = mpi.COMM.gather(prob) # if mpi.is_root(): # np.vstack(prob) # #Train # mpi.COMM.Bcast(self._nn) # mpi.distribute(prob) DS = ClassificationDataSet( Xtrain_feats.shape[1], 1, nb_classes=2 ) #for i in range(Xtrain_feats.shape[0]): # DS.addSample( Xtrain_feats[i,:], [self._Ytrain[i]] ) DS.setField('input', Xtrain_feats) DS.setField('target', self._Ytrain[:,np.newaxis]) DS._convertToOneOfMany() self._nn = buildNetwork(DS.indim, 10, DS.outdim, outclass=SoftmaxLayer, fast=True) self._nn_trainer = BackpropTrainer( self._nn, dataset=DS, momentum=0.1, verbose=True, weightdecay=gamma, learningrate=0.01, lrdecay=1.0) self._nn_trainer.trainOnDataset(DS,epochs=8) self._nn_trainer = BackpropTrainer( self._nn, dataset=DS, momentum=0.1, verbose=True, weightdecay=gamma, learningrate=0.001, lrdecay=1.0) self._nn_trainer.trainOnDataset(DS,epochs=8) self._nn_trainer = BackpropTrainer( self._nn, dataset=DS, momentum=0.1, verbose=True, weightdecay=gamma, learningrate=0.0001, lrdecay=1.0) self._nn_trainer.trainOnDataset(DS,epochs=5) return self._nn def Accuracy(self, X, Y, special_bias = None): X_feats = np.ascontiguousarray(np.hstack((X[self.feat_list[i]] for i in range(len(self.feat_list))))) X_feats -= self.m X_feats /= self.std if special_bias != None: X_feats = np.ascontiguousarray(np.hstack((X_feats, special_bias))) if self._type=='linsvm' or self._type=='logreg' or self._type=='logreg_atwv': self.test_accu = classifier.Evaluator.accuracy(Y, np.dot(X_feats,self.w)+self.b) elif self._type=='nn_atwv': pred = get_predictions_nn(X_feats, self._weights_nn, arch=[10])[0] pred[:,0] = 0.5 self.test_accu = classifier.Evaluator.accuracy(Y, pred) else: DS = ClassificationDataSet( X_feats.shape[1], 1, nb_classes=2 ) #for i in range(X_feats.shape[0]): # DS.addSample( X_feats[i,:], [Y[i]] ) DS.setField('input', X_feats) DS.setField('target', Y[:,np.newaxis]) DS._convertToOneOfMany() predict,targts = self._nn_trainer.testOnClassData(DS, verbose=True,return_targets=True) self.test_accu = np.sum(np.array(predict)==np.array(targts))/float(len(targts)) return self.test_accu def loss_multiclass_logreg(self, X, Y, special_bias=None): X_feats=np.hstack((X[self.feat_list[i]] for i in range(len(self.feat_list)))) X_feats -= self.m X_feats /= self.std if special_bias != None: X_feats = np.ascontiguousarray(np.hstack((X_feats, special_bias))) return loss_multiclass_logreg(Y, X_feats, (self.w,self.b)) def loss_multiclass_nn(self, X, Y, special_bias=None): X_feats = np.ascontiguousarray(np.hstack((X[self.feat_list[i]] for i in range(len(self.feat_list))))) X_feats -= self.m X_feats /= self.std if special_bias != None: X_feats = np.ascontiguousarray(np.hstack((X_feats, special_bias))) return loss_multiclass_nn(X_feats, Y, self._weights_nn, self._arch) def get_predictions_logreg(self, X, special_bias=None): X_feats=np.hstack((X[self.feat_list[i]] for i in range(len(self.feat_list)))) X_feats -= self.m X_feats /= self.std if special_bias != None: X_feats = np.ascontiguousarray(np.hstack((X_feats, special_bias))) return get_predictions_logreg(X_feats, (self.w,self.b)) def get_predictions_nn_old(self, X, special_bias=None): X_feats = np.ascontiguousarray(np.hstack((X[self.feat_list[i]] for i in range(len(self.feat_list))))) X_feats -= self.m X_feats /= self.std if special_bias != None: X_feats = np.ascontiguousarray(np.hstack((X_feats, special_bias))) DS = ClassificationDataSet( X_feats.shape[1], 1, nb_classes=2 ) #for i in range(X_feats.shape[0]): # DS.addSample( X_feats[i,:], [0.0] ) DS.setField('input', X_feats) DS.setField('target', np.zeros((X_feats.shape[0],1))) DS._convertToOneOfMany() prob = self._nn.activateOnDataset(DS) prob = mpi.COMM.gather(prob) if mpi.is_root(): return np.vstack(prob) else: return np.zeros((0)) def get_predictions_nn(self, X, special_bias=None): X_feats = np.ascontiguousarray(np.hstack((X[self.feat_list[i]] for i in range(len(self.feat_list))))) X_feats -= self.m X_feats /= self.std if special_bias != None: X_feats = np.ascontiguousarray(np.hstack((X_feats, special_bias))) prob = get_predictions_nn(X_feats, self._weights_nn, self._arch)[0] prob = mpi.COMM.gather(prob) if mpi.is_root(): return np.vstack(prob) else: return np.zeros((0))
net.addModule(hidden_layer) net.addOutputModule(output_layer) net.addConnection( FullConnection(input_layer, hidden_layer, name='in_to_hidden')) net.addConnection( FullConnection(hidden_layer, output_layer, name='hidden_to_out')) net.sortModules() #backpropagation trainer = BackpropTrainer(net, dataset=trndata, momentum=0.1, verbose=True, weightdecay=0.01) #error checking part for i in range(10): trainer.trainEpochs(1) trnresult = percentError(trainer.testOnClassData(), trndata['target']) tstresult = percentError(trainer.testOnClassData(dataset=tstdata), tstdata['target']) trigram_file = open('trigram.txt', 'w') trigram_file.writelines(["%s\n" % item for item in sorted_list]) word_file = open('word_list', 'w') word_file.writelines(["%s\n" % item for item in input_list]) word_file.close() trigram_file.close() text_file.close()
net.addOutputModule(output_layer) net.addConnection(FullConnection(input_layer, hidden_layer, name='in_to_hidden')) net.addConnection(FullConnection(hidden_layer, output_layer, name='hidden_to_out')) net.sortModules() #backpropagation trainer = BackpropTrainer(net, dataset=trndata, momentum=0.1, verbose=True, weightdecay=0.01) #error checking part for i in range(10): trainer.trainEpochs(1) trnresult = percentError(trainer.testOnClassData(), trndata['target']) tstresult = percentError(trainer.testOnClassData(dataset=tstdata), tstdata['target']) trigram_file = open('trigram.txt', 'w') trigram_file.writelines(["%s\n" % item for item in sorted_list]) word_file = open('word_list', 'w') word_file.writelines(["%s\n" % item for item in input_list]) word_file.close() trigram_file.close() text_file.close()
leg = plt.legend([dot1, dot2], ['0','1']) ax = plt.gca().add_artist(leg) plt.show() """ elif sys.argv[1] == 'nn': DS = ClassificationDataSet(165) training_data = np.array(training_data) training_target = np.vstack(np.array(training_target)) print len(training_data[0]) #print len(training_target[0]) assert(training_data.shape[0] == training_target.shape[0]) DS.setField('input', training_data) DS.setField('target', training_target) tstdata, trndata = DS.splitWithProportion(0.15) hidden_layer_neurons = (DS.indim+DS.outdim)/2 rnn = buildNetwork(DS.indim,hidden_layer_neurons,DS.outdim,hiddenclass=LSTMLayer,outclass=SigmoidLayer,outputbias=False,recurrent=True) #print hidden_layer_neurons # define a training method trainer = BackpropTrainer(rnn,dataset=trndata, verbose=True) trainer.trainUntilConvergence(verbose = True, validationProportion = 0.3, maxEpochs = 1000, continueEpochs = 10) print 'Percent Error on Test dataset: ' , percentError(trainer.testOnClassData(tstdata, verbose=True), tstdata['target'] ) print 'Percent Error on Test dataset: ' , percentError(trainer.testOnClassData(tstdata, verbose=True), tstdata['target'] ) #print 'Percent Error on Training dataset: ' , percentError(trainer.testOnClassData(trndata), trndata['target'] ) else: print("Current classifier algorithms available: svm, knn, dt, kmeans, nn") sys.exit(1) if running_tests: print "\nMax Accuracy: " + str(max(accuracy_arr))