def network(dataset, input_list):
num_words = len(input_list)
#dividing the dataset into training and testing data
tstdata, trndata = dataset.splitWithProportion(0.25)
#building the network
net = RecurrentNetwork()
input_layer1 = LinearLayer(num_words, name='input_layer1')
input_layer2 = LinearLayer(num_words, name='input_layer2')
hidden_layer = TanhLayer(num_words, name='hidden_layer')
output_layer = SoftmaxLayer(num_words, name='output_layer')
net.addInputModule(input_layer1)
net.addInputModule(input_layer2)
net.addModule(hidden_layer)
net.addOutputModule(output_layer)
net.addConnection(FullConnection(input_layer1,
hidden_layer,
name='in1_to_hidden'))
net.addConnection(FullConnection(input_layer2, hidden_layer,
name='in2_to_hidden'))
net.addConnection(FullConnection(hidden_layer,
output_layer,
name='hidden_to_output'))
net.addConnection(FullConnection(input_layer1,
output_layer,
name='in1_to_out'))
net.addConnection(FullConnection(input_layer2,
output_layer,
name='in2_to_out'))
net.sortModules()
#backpropagation
trainer = BackpropTrainer(net, dataset=trndata,
momentum=0.1,
verbose=True,
weightdecay=0.01)
#error checking part
for i in range(10):
trainer.trainEpochs(1)
trnresult = percentError(trainer.testOnClassData(), trndata['target'])
tstresult = percentError(trainer.testOnClassData(dataset=tstdata),
tstdata['target'])
print "epoch: %4d" % trainer.totalepochs
print " train error: %5.10f%%" % trnresult
print " test error: %5.10f%%" % tstresult
return net
def training_and_testing():
nn = init_neural_network()
training = learning.get_labeled_data(
'%strain-images-idx3-ubyte.gz' % (database_folder),
'%strain-labels-idx1-ubyte.gz' % (database_folder),
'%strainig' % (database_folder))
test = learning.get_labeled_data(
'%st10k-images-idx3-ubyte.gz' % (database_folder),
'%st10k-labels-idx1-ubyte.gz' % (database_folder),
'%stest' % (database_folder))
FEATURES = N_INPUT_LAYER
print("Caracteristicas a analizar: %i" % FEATURES)
testdata = ClassificationDataSet(FEATURES, 1, nb_classes=OUTPUT_LAYER)
trainingdata = ClassificationDataSet(FEATURES, 1, nb_classes=OUTPUT_LAYER)
for i in range(len(test['data'])):
testdata.addSample(test['data'][i], test['label'][i])
for j in range(len(training['data'])):
trainingdata.addSample(training['data'][j], training['label'][j])
trainingdata._convertToOneOfMany()
testdata._convertToOneOfMany()
trainer = BackpropTrainer(nn,
dataset=trainingdata,
momentum=MOMENTUM,
verbose=True,
weightdecay=W_DECAY,
learningrate=L_RATE,
lrdecay=L_DECAY)
for i in range(EPOCHS):
trainer.trainEpochs(1)
trnresult = percentError(trainer.testOnClassData(),
trainingdata['class'])
tstresult = percentError(trainer.testOnClassData(dataset=testdata),
testdata['class'])
print("epoch: %4d" % trainer.totalepochs,
" train error: %5.2f%%" % trnresult,
" test error: %5.2f%%" % tstresult)
return nn
def network(dataset, input_list):
num_words = len(input_list)
#dividing the dataset into training and testing data
tstdata, trndata = dataset.splitWithProportion(0.25)
#building the network
net = RecurrentNetwork()
input_layer1 = LinearLayer(num_words, name='input_layer1')
input_layer2 = LinearLayer(num_words, name='input_layer2')
hidden_layer = TanhLayer(num_words, name='hidden_layer')
output_layer = SoftmaxLayer(num_words, name='output_layer')
net.addInputModule(input_layer1)
net.addInputModule(input_layer2)
net.addModule(hidden_layer)
net.addOutputModule(output_layer)
net.addConnection(
FullConnection(input_layer1, hidden_layer, name='in1_to_hidden'))
net.addConnection(
FullConnection(input_layer2, hidden_layer, name='in2_to_hidden'))
net.addConnection(
FullConnection(hidden_layer, output_layer, name='hidden_to_output'))
net.addConnection(
FullConnection(input_layer1, output_layer, name='in1_to_out'))
net.addConnection(
FullConnection(input_layer2, output_layer, name='in2_to_out'))
net.sortModules()
#backpropagation
trainer = BackpropTrainer(net,
dataset=trndata,
momentum=0.1,
verbose=True,
weightdecay=0.01)
#error checking part
for i in range(10):
trainer.trainEpochs(1)
trnresult = percentError(trainer.testOnClassData(), trndata['target'])
tstresult = percentError(trainer.testOnClassData(dataset=tstdata),
tstdata['target'])
print "epoch: %4d" % trainer.totalepochs
print " train error: %5.10f%%" % trnresult
print " test error: %5.10f%%" % tstresult
return net
def training_and_testing():
nn= init_neural_network()
training = learning.get_labeled_data('%strain-images-idx3-ubyte.gz'%(database_folder),
'%strain-labels-idx1-ubyte.gz'%(database_folder)
,'%strainig'%(database_folder))
test = learning.get_labeled_data('%st10k-images-idx3-ubyte.gz'%(database_folder),
'%st10k-labels-idx1-ubyte.gz'%(database_folder),
'%stest'%(database_folder))
FEATURES = N_INPUT_LAYER
print("Caracteristicas a analizar: %i"%FEATURES)
testdata = ClassificationDataSet(FEATURES,1,nb_classes=OUTPUT_LAYER)
trainingdata = ClassificationDataSet(FEATURES,1,nb_classes=OUTPUT_LAYER)
for i in range(len(test['data'])):
testdata.addSample(test['data'][i],test['label'][i])
for j in range(len(training['data'])):
trainingdata.addSample(training['data'][j],training['label'][j])
trainingdata._convertToOneOfMany()
testdata._convertToOneOfMany()
trainer = BackpropTrainer(nn,dataset=trainingdata,momentum=MOMENTUM,verbose=True,
weightdecay=W_DECAY,learningrate=L_RATE,lrdecay=L_DECAY)
for i in range(EPOCHS):
trainer.trainEpochs(1)
trnresult = percentError(trainer.testOnClassData(),
trainingdata['class'])
tstresult = percentError(trainer.testOnClassData(
dataset=testdata), testdata['class'])
print("epoch: %4d" % trainer.totalepochs,
" train error: %5.2f%%" % trnresult,
" test error: %5.2f%%" % tstresult)
return nn
from pybrain.tools.shortcuts import buildNetwork
from pybrain.structure import SoftmaxLayer
# 输入数据是 19维，输出是两维，隐层设置为5层
# 输出层使用Softmax激活，其他：学习率(learningrate=0.01),学习率衰减(lrdecay=1.0，每次训练一步学习率乘以)，
# 详细（verbose=False）动量因子(momentum=0最后时步的梯度？），权值衰减？(weightdecay=0.0)
n_h = 5
net = buildNetwork(19, n_h, 2, outclass=SoftmaxLayer)
# Step 2 : 构建前馈网络标准BP算法
from pybrain.supervised import BackpropTrainer
trainer_sd = BackpropTrainer(net, traindata)
# # 或者使用累积BP算法,训练次数50次
# trainer_ac = BackpropTrainer(net, traindata, batchlearning=True)
# trainer_ac.trainEpochs(50)
# err_train, err_valid = trainer_ac.trainUntilConvergence(maxEpochs=50)
for i in range(50): # 训练50次，每及测试结果次打印训练结果
trainer_sd.trainEpochs(1) # 训练网络一次,
# 引入训练误差和测试误差
from pybrain.utilities import percentError
trainresult = percentError(trainer_sd.testOnClassData(),
traindata['class'])
testresult = percentError(trainer_sd.testOnClassData(dataset=testdata),
testdata['class'])
# 打印错误率
print('Epoch: %d', trainer_sd.totalepochs, 'train error: ', trainresult,
'test error: ', testresult)
class Classifier:
def __init__(self,Xtrain,Ytrain):
self._Xtrain=Xtrain
self._Ytrain=Ytrain
self.features=Xtrain.keys()
def Train(self,feat_list=None,type='logreg',gamma=0.0,domeanstd=True,special_bias=None,add_bias=True, weight=None, class_instance=None, method='sigmoid',factor=10.0,arch=[10],
cv_feats=None, cv_special_bias=None,cv_class_instance=None):
if feat_list==None:
feat_list=self.features
self.feat_list=feat_list
self._gamma=gamma
self._type=type
self._special_bias = special_bias
self._add_bias = add_bias
Xtrain_feats = np.ascontiguousarray(np.hstack((self._Xtrain[feat] for feat in feat_list)))
self.m, self.std = classifier.feature_meanstd(Xtrain_feats)
if domeanstd==False: #hacky, overwrite the things we computed
self.m[:] = 0
self.std[:] = 1
Xtrain_feats -= self.m
Xtrain_feats /= self.std
if special_bias != None:
Xtrain_feats = np.ascontiguousarray(np.hstack((Xtrain_feats, special_bias)))
#CV
if cv_feats!=None:
cv_feats = np.ascontiguousarray(np.hstack((cv_feats[feat] for feat in feat_list)))
cv_feats -= self.m
cv_feats /= self.std
if special_bias != None:
cv_feats = np.ascontiguousarray(np.hstack((cv_feats, cv_special_bias)))
'''Classifier stage'''
if type=='linsvm':
self.w, self.b = classifier.svm_onevsall(Xtrain_feats, self._Ytrain, self._gamma, weight = weight, special_bias=special_bias, add_bias=add_bias)
return (self.w,self.b)
elif type=='logreg':
self.w, self.b = l2logreg_onevsall(Xtrain_feats, self._Ytrain, self._gamma, weight = weight, special_bias=special_bias, add_bias=add_bias)
return (self.w,self.b)
elif type=='logreg_atwv':
self.w, self.b = Train_atwv(Xtrain_feats,class_instance=class_instance,weight=weight,special_bias=special_bias, add_bias=add_bias, method=method,
factor=factor, gamma=self._gamma, cv_class_instance=cv_class_instance, cv_feats=cv_feats)
elif type=='nn_atwv':
self._arch = arch
self._weights_nn = Train_atwv_nn(Xtrain_feats,class_instance=class_instance,weight=weight,special_bias=special_bias, add_bias=add_bias,
arch=self._arch, method=method, factor=factor, gamma=self._gamma, cv_class_instance=cv_class_instance, cv_feats=cv_feats)
#self._weights_nn = Train_atwv_nn(Xtrain_feats,class_instance=class_instance,weight=self._weights_nn,special_bias=special_bias, add_bias=add_bias,
# arch=self._arch, method=method, factor=factor*10.0)
elif type=='nn_debug':
if mpi.COMM.Get_size() > 1:
print 'Warning!!! Running NN training with MPI with more than one Node!'
#FIXME: Collect X and Y at root to avoid this
# prob = mpi.COMM.gather(prob)
# if mpi.is_root():
# np.vstack(prob)
# #Train
# mpi.COMM.Bcast(self._nn)
# mpi.distribute(prob)
DS = ClassificationDataSet( Xtrain_feats.shape[1], 1, nb_classes=2 )
#for i in range(Xtrain_feats.shape[0]):
# DS.addSample( Xtrain_feats[i,:], [self._Ytrain[i]] )
DS.setField('input', Xtrain_feats)
DS.setField('target', self._Ytrain[:,np.newaxis])
DS._convertToOneOfMany()
self._nn = buildNetwork(DS.indim, 10, DS.outdim, outclass=SoftmaxLayer, fast=True)
self._nn_trainer = BackpropTrainer( self._nn, dataset=DS, momentum=0.1, verbose=True, weightdecay=gamma, learningrate=0.01, lrdecay=1.0)
self._nn_trainer.trainOnDataset(DS,epochs=8)
self._nn_trainer = BackpropTrainer( self._nn, dataset=DS, momentum=0.1, verbose=True, weightdecay=gamma, learningrate=0.001, lrdecay=1.0)
self._nn_trainer.trainOnDataset(DS,epochs=8)
self._nn_trainer = BackpropTrainer( self._nn, dataset=DS, momentum=0.1, verbose=True, weightdecay=gamma, learningrate=0.0001, lrdecay=1.0)
self._nn_trainer.trainOnDataset(DS,epochs=5)
return self._nn
def Accuracy(self, X, Y, special_bias = None):
X_feats = np.ascontiguousarray(np.hstack((X[self.feat_list[i]] for i in range(len(self.feat_list)))))
X_feats -= self.m
X_feats /= self.std
if special_bias != None:
X_feats = np.ascontiguousarray(np.hstack((X_feats, special_bias)))
if self._type=='linsvm' or self._type=='logreg' or self._type=='logreg_atwv':
self.test_accu = classifier.Evaluator.accuracy(Y, np.dot(X_feats,self.w)+self.b)
elif self._type=='nn_atwv':
pred = get_predictions_nn(X_feats, self._weights_nn, arch=[10])[0]
pred[:,0] = 0.5
self.test_accu = classifier.Evaluator.accuracy(Y, pred)
else:
DS = ClassificationDataSet( X_feats.shape[1], 1, nb_classes=2 )
#for i in range(X_feats.shape[0]):
# DS.addSample( X_feats[i,:], [Y[i]] )
DS.setField('input', X_feats)
DS.setField('target', Y[:,np.newaxis])
DS._convertToOneOfMany()
predict,targts = self._nn_trainer.testOnClassData(DS, verbose=True,return_targets=True)
self.test_accu = np.sum(np.array(predict)==np.array(targts))/float(len(targts))
return self.test_accu
def loss_multiclass_logreg(self, X, Y, special_bias=None):
X_feats=np.hstack((X[self.feat_list[i]] for i in range(len(self.feat_list))))
X_feats -= self.m
X_feats /= self.std
if special_bias != None:
X_feats = np.ascontiguousarray(np.hstack((X_feats, special_bias)))
return loss_multiclass_logreg(Y, X_feats, (self.w,self.b))
def loss_multiclass_nn(self, X, Y, special_bias=None):
X_feats = np.ascontiguousarray(np.hstack((X[self.feat_list[i]] for i in range(len(self.feat_list)))))
X_feats -= self.m
X_feats /= self.std
if special_bias != None:
X_feats = np.ascontiguousarray(np.hstack((X_feats, special_bias)))
return loss_multiclass_nn(X_feats, Y, self._weights_nn, self._arch)
def get_predictions_logreg(self, X, special_bias=None):
X_feats=np.hstack((X[self.feat_list[i]] for i in range(len(self.feat_list))))
X_feats -= self.m
X_feats /= self.std
if special_bias != None:
X_feats = np.ascontiguousarray(np.hstack((X_feats, special_bias)))
return get_predictions_logreg(X_feats, (self.w,self.b))
def get_predictions_nn_old(self, X, special_bias=None):
X_feats = np.ascontiguousarray(np.hstack((X[self.feat_list[i]] for i in range(len(self.feat_list)))))
X_feats -= self.m
X_feats /= self.std
if special_bias != None:
X_feats = np.ascontiguousarray(np.hstack((X_feats, special_bias)))
DS = ClassificationDataSet( X_feats.shape[1], 1, nb_classes=2 )
#for i in range(X_feats.shape[0]):
# DS.addSample( X_feats[i,:], [0.0] )
DS.setField('input', X_feats)
DS.setField('target', np.zeros((X_feats.shape[0],1)))
DS._convertToOneOfMany()
prob = self._nn.activateOnDataset(DS)
prob = mpi.COMM.gather(prob)
if mpi.is_root():
return np.vstack(prob)
else:
return np.zeros((0))
def get_predictions_nn(self, X, special_bias=None):
X_feats = np.ascontiguousarray(np.hstack((X[self.feat_list[i]] for i in range(len(self.feat_list)))))
X_feats -= self.m
X_feats /= self.std
if special_bias != None:
X_feats = np.ascontiguousarray(np.hstack((X_feats, special_bias)))
prob = get_predictions_nn(X_feats, self._weights_nn, self._arch)[0]
prob = mpi.COMM.gather(prob)
if mpi.is_root():
return np.vstack(prob)
else:
return np.zeros((0))
net.addModule(hidden_layer)
net.addOutputModule(output_layer)
net.addConnection(
FullConnection(input_layer, hidden_layer, name='in_to_hidden'))
net.addConnection(
FullConnection(hidden_layer, output_layer, name='hidden_to_out'))
net.sortModules()
#backpropagation
trainer = BackpropTrainer(net,
dataset=trndata,
momentum=0.1,
verbose=True,
weightdecay=0.01)
#error checking part
for i in range(10):
trainer.trainEpochs(1)
trnresult = percentError(trainer.testOnClassData(), trndata['target'])
tstresult = percentError(trainer.testOnClassData(dataset=tstdata),
tstdata['target'])
trigram_file = open('trigram.txt', 'w')
trigram_file.writelines(["%s\n" % item for item in sorted_list])
word_file = open('word_list', 'w')
word_file.writelines(["%s\n" % item for item in input_list])
word_file.close()
trigram_file.close()
text_file.close()
net.addOutputModule(output_layer)
net.addConnection(FullConnection(input_layer,
hidden_layer,
name='in_to_hidden'))
net.addConnection(FullConnection(hidden_layer,
output_layer,
name='hidden_to_out'))
net.sortModules()
#backpropagation
trainer = BackpropTrainer(net, dataset=trndata,
momentum=0.1,
verbose=True,
weightdecay=0.01)
#error checking part
for i in range(10):
trainer.trainEpochs(1)
trnresult = percentError(trainer.testOnClassData(), trndata['target'])
tstresult = percentError(trainer.testOnClassData(dataset=tstdata),
tstdata['target'])
trigram_file = open('trigram.txt', 'w')
trigram_file.writelines(["%s\n" % item for item in sorted_list])
word_file = open('word_list', 'w')
word_file.writelines(["%s\n" % item for item in input_list])
word_file.close()
trigram_file.close()
text_file.close()
leg = plt.legend([dot1, dot2], ['0','1'])
ax = plt.gca().add_artist(leg)
plt.show()
"""
elif sys.argv[1] == 'nn':
DS = ClassificationDataSet(165)
training_data = np.array(training_data)
training_target = np.vstack(np.array(training_target))
print len(training_data[0])
#print len(training_target[0])
assert(training_data.shape[0] == training_target.shape[0])
DS.setField('input', training_data)
DS.setField('target', training_target)
tstdata, trndata = DS.splitWithProportion(0.15)
hidden_layer_neurons = (DS.indim+DS.outdim)/2
rnn = buildNetwork(DS.indim,hidden_layer_neurons,DS.outdim,hiddenclass=LSTMLayer,outclass=SigmoidLayer,outputbias=False,recurrent=True)
#print hidden_layer_neurons
# define a training method
trainer = BackpropTrainer(rnn,dataset=trndata, verbose=True)
trainer.trainUntilConvergence(verbose = True, validationProportion = 0.3, maxEpochs = 1000, continueEpochs = 10)
print 'Percent Error on Test dataset: ' , percentError(trainer.testOnClassData(tstdata, verbose=True), tstdata['target'] )
print 'Percent Error on Test dataset: ' , percentError(trainer.testOnClassData(tstdata, verbose=True), tstdata['target'] )
#print 'Percent Error on Training dataset: ' , percentError(trainer.testOnClassData(trndata), trndata['target'] )
else:
print("Current classifier algorithms available: svm, knn, dt, kmeans, nn")
sys.exit(1)
if running_tests:
print "\nMax Accuracy: " + str(max(accuracy_arr))
