class NNetwork: def __init__(self): self.ds = ClassificationDataSet(7, 1, nb_classes=8) #8 since we have 8 gestures, 7 since we have 7 features def add_data(self, training_data): for gesture in training_data: self.ds.addSample(gesture[1], gesture[0]) #a method to add all the training data we have def newData(self, training_data): #a method for replacing the data already existing and adding data from scratch self.ds = ClassificationDataSet(7, 1, nb_classes=8) for gesture in training_data: self.ds.addSample(gesture[1], gesture[0]) def train(self, shouldPrint): tstdata, trndata = self.ds.splitWithProportion(0.2) #splits the data into training and verification data trndata._convertToOneOfMany() tstdata._convertToOneOfMany() self.fnn = buildNetwork(trndata.indim, 64, trndata.outdim, outclass=SoftmaxLayer) #builds a network with 64 hidden neurons self.trainer = BackpropTrainer(self.fnn, dataset=trndata, momentum=0.1, learningrate=0.01, verbose=True, weightdecay=0.1) #uses the backpropagation algorithm self.trainer.trainUntilConvergence(dataset=trndata, maxEpochs=100, verbose=True, continueEpochs=10, validationProportion=0.20) #early stopping with 20% as testing data trnresult = percentError( self.trainer.testOnClassData(), trndata['class'] ) tstresult = percentError( self.trainer.testOnClassData(dataset=tstdata ), tstdata['class'] ) if shouldPrint: print "epoch: %4d" % self.trainer.totalepochs, " train error: %5.2f%%" % trnresult, " test error: %5.2f%%" % tstresult def activate(self, data): #tests a particular data point (feature vector) return self.fnn.activate(data)
def measuredLearning(ds):
trndata,tstdata = splitData(ds,.025)
#build network
###
# This network has no hidden layters, you might need to add some
###
fnn = buildNetwork( trndata.indim, 22, trndata.outdim, outclass=SoftmaxLayer )
trainer = BackpropTrainer( fnn, verbose=True,dataset=trndata)
####
# Alter this to figure out how many runs you want. Best to start small and be sure that you see learning.
# Before you ramp it up.
###
for i in range(150):
trainer.trainEpochs(5)
trnresult = percentError(trainer.testOnClassData(),trndata['class'] )
tstresult = percentError( trainer.testOnClassData(
dataset=tstdata ), tstdata['class'] )
print "epoch: %4d" % trainer.totalepochs, \
" train error: %5.2f%%" % trnresult, \
" test error: %5.2f%%" % tstresult
if(trnresult<.5):
return
def big_training(np_data, num_nets=1, num_epoch=20, net_builder=net_full, train_size=.1, testing=False):
sss = cross_validation.StratifiedShuffleSplit(np_data[:,:1].ravel(), n_iter=num_nets , test_size=1-train_size, random_state=3476)
nets=[None for net_ind in range(num_nets)]
trainaccu=[[0 for i in range(num_epoch)] for net_ind in range(num_nets)]
testaccu=[[0 for i in range(num_epoch)] for net_ind in range(num_nets)]
net_ind=0
for train_index, test_index in sss:
print ('%s Building %d. network.' %(time.ctime(), net_ind+1))
#print("TRAIN:", len(train_index), "TEST:", len(test_index))
trainset = ClassificationDataSet(np_data.shape[1] - 1, 1)
trainset.setField('input', np_data[train_index,1:]/100-.6)
trainset.setField('target', np_data[train_index,:1])
trainset._convertToOneOfMany( )
trainlabels = trainset['class'].ravel().tolist()
if testing:
testset = ClassificationDataSet(np_data.shape[1] - 1, 1)
testset.setField('input', np_data[test_index,1:]/100-.6)
testset.setField('target', np_data[test_index,:1])
testset._convertToOneOfMany( )
testlabels = testset['class'].ravel().tolist()
nets[net_ind] = net_builder()
trainer = BackpropTrainer(nets[net_ind], trainset)
for i in range(num_epoch):
for ii in range(3):
err = trainer.train()
print ('%s Epoch %d: Network trained with error %f.' %(time.ctime(), i+1, err))
trainaccu[net_ind][i]=accuracy_score(trainlabels,trainer.testOnClassData())
print ('%s Epoch %d: Train accuracy is %f' %(time.ctime(), i+1, trainaccu[net_ind][i]))
print ([sum([trainaccu[y][i]>tres for y in range(net_ind+1)]) for tres in [0,.1,.2,.3,.4,.5,.6]])
if testing:
testaccu[net_ind][i]=accuracy_score(testlabels,trainer.testOnClassData(testset))
print ('%s Epoch %d: Test accuracy is %f' %(time.ctime(), i+1, testaccu[net_ind][i]))
NetworkWriter.writeToFile(nets[net_ind], 'nets/'+net_builder.__name__+str(net_ind)+'.xml')
net_ind +=1
return [nets, trainaccu, testaccu]
def createnetwork(n_hoglist,n_classlist,n_classnum,n_hiddensize=100):
n_inputdim=len(n_hoglist[0])
n_alldata = ClassificationDataSet(n_inputdim,1, nb_classes=n_classnum)
for i in range(len(n_hoglist)):
n_input = n_hoglist[i]
n_class = n_classlist[i]
n_alldata.addSample(n_input, [n_class])
n_tstdata, n_trndata = n_alldata.splitWithProportion( 0.25 )
n_trndata._convertToOneOfMany( )
n_tstdata._convertToOneOfMany( )
print "Number of training patterns: ", len(n_trndata)
print "Input and output dimensions: ", n_trndata.indim, n_trndata.outdim
print "First sample (input, target, class):"
print n_trndata['input'][0], n_trndata['target'][0], n_trndata['class'][0]
n_fnn = buildNetwork(n_trndata.indim,n_hiddensize, n_trndata.outdim, outclass=SoftmaxLayer)
n_trainer = BackpropTrainer(n_fnn, dataset=n_trndata, momentum=0.1, verbose=True, weightdecay=0.01)
n_result = 1
while n_result > 0.1:
print n_result
n_trainer.trainEpochs(1)
n_trnresult = percentError(n_trainer.testOnClassData(),
n_trndata['class'])
n_tstresult = percentError(n_trainer.testOnClassData(
dataset=n_tstdata), n_tstdata['class'])
print "epoch: %4d" % n_trainer.totalepochs, \
" train error: %5.2f%%" % n_trnresult, \
" test error: %5.2f%%" % n_tstresult
n_result = n_tstresult
def main():
images, labels = load_labeled_training(flatten=True)
images = standardize(images)
# images, labels = load_pca_proj(K=100)
shuffle_in_unison(images, labels)
ds = ClassificationDataSet(images.shape[1], 1, nb_classes=7)
for i, l in zip(images, labels):
ds.addSample(i, [l - 1])
# ds._convertToOneOfMany()
test, train = ds.splitWithProportion(0.2)
test._convertToOneOfMany()
train._convertToOneOfMany()
net = shortcuts.buildNetwork(train.indim, 1000, train.outdim, outclass=SoftmaxLayer)
trainer = BackpropTrainer(net, dataset=train, momentum=0.1, learningrate=0.01, weightdecay=0.05)
# trainer = RPropMinusTrainer(net, dataset=train)
# cv = validation.CrossValidator(trainer, ds)
# print cv.validate()
net.randomize()
tr_labels_2 = net.activateOnDataset(train).argmax(axis=1)
trnres = percentError(tr_labels_2, train["class"])
# trnres = percentError(trainer.testOnClassData(dataset=train), train['class'])
testres = percentError(trainer.testOnClassData(dataset=test), test["class"])
print "Training error: %.10f, Test error: %.10f" % (trnres, testres)
print "Iters: %d" % trainer.totalepochs
for i in range(100):
trainer.trainEpochs(10)
trnres = percentError(trainer.testOnClassData(dataset=train), train["class"])
testres = percentError(trainer.testOnClassData(dataset=test), test["class"])
trnmse = trainer.testOnData(dataset=train)
testmse = trainer.testOnData(dataset=test)
print "Iteration: %d, Training error: %.5f, Test error: %.5f" % (trainer.totalepochs, trnres, testres)
print "Training MSE: %.5f, Test MSE: %.5f" % (trnmse, testmse)
class Classifier():
def __init__(self, testing = False):
self.training_set, self.test_set = split_samples(0.5 if testing else 1.0)
self.net = buildNetwork( self.training_set.indim, self.training_set.outdim, outclass=SoftmaxLayer )
self.trainer = BackpropTrainer( self.net, dataset=self.training_set, momentum=0.1, verbose=True, weightdecay=0.01)
self.train()
def train(self):
self.trainer.trainEpochs( EPOCHS )
trnresult = percentError( self.trainer.testOnClassData(),
self.training_set['class'] )
print " train error: %5.2f%%" % trnresult
def classify(self, file):
strengths = self.net.activate(process_sample(*load_sample(file)))
print strengths
best_match = None
strength = 0.0
for i,s in enumerate(strengths):
if s > strength:
best_match = i
strength = s
return SOUNDS[best_match]
def test(self):
tstresult = percentError( self.trainer.testOnClassData(
dataset=self.test_set ), self.test_set['class'] )
print " test error: %5.2f%%" % tstresult
def trainNetwork(epochs, rate, trndata, tstdata, network=None):
'''
epochs: number of iterations to run on dataset
trndata: pybrain ClassificationDataSet
tstdat: pybrain ClassificationDataSet
network: filename of saved pybrain network, or None
'''
if network is None:
net = buildNetwork(400, 25, 25, 9, bias=True, hiddenclass=SigmoidLayer, outclass=SigmoidLayer)
else:
net = NetworkReader.readFrom(network)
print "Number of training patterns: ", len(trndata)
print "Input and output dimensions: ", trndata.indim, trndata.outdim
print "First sample input:"
print trndata['input'][0]
print ""
print "First sample target:", trndata['target'][0]
print "First sample class:", trndata.getClass(int(trndata['class'][0]))
print ""
trainer = BackpropTrainer(net, dataset=trndata, learningrate=rate)
for i in range(epochs):
trainer.trainEpochs(1)
trnresult = percentError(trainer.testOnClassData(), trndata['class'])
tstresult = percentError(trainer.testOnClassData(dataset=tstdata), tstdata['class'])
print "epoch: %4d" % trainer.totalepochs, " train error: %5.2f%%" % trnresult, " test error: %5.2f%%" % tstresult
return net
def train(self): print "Enter the number of times to train, -1 means train until convergence:" t = int(raw_input()) print "Training the Neural Net" print "self.net.indim = "+str(self.net.indim) print "self.train_data.indim = "+str(self.train_data.indim) trainer = BackpropTrainer(self.net, dataset=self.train_data, momentum=0.1, verbose=True, weightdecay=0.01) if t == -1: trainer.trainUntilConvergence() else: for i in range(t): trainer.trainEpochs(1) trnresult = percentError( trainer.testOnClassData(), self.train_data['class']) # print self.test_data tstresult = percentError( trainer.testOnClassData(dataset=self.test_data), self.test_data['class'] ) print "epoch: %4d" % trainer.totalepochs, \ " train error: %5.2f%%" % trnresult, \ " test error: %5.2f%%" % tstresult if i % 10 == 0 and i > 1: print "Saving Progress... Writing to a file" NetworkWriter.writeToFile(self.net, self.path) print "Done training... Writing to a file" NetworkWriter.writeToFile(self.net, self.path) return trainer
def main():
trndata, tstdata = createDS()
for repeat in xrange(repeats):
iter_trn_results = []
iter_tst_results = []
nn = createNNLong(trndata)
hiddenAstrocyteLayer, outputAstrocyteLayer = associateAstrocyteLayers(nn)
trainer = BackpropTrainer(nn, dataset=trndata, learningrate=0.01,
momentum=0.1, verbose=False, weightdecay=0.0)
for grand_iter in xrange(iterations):
trainer.trainEpochs(1)
trnresult = percentError(trainer.testOnClassData(), trndata['class'])
iter_trn_results.append(trnresult)
tstresult = percentError(trainer.testOnClassData(dataset=tstdata), tstdata['class'])
iter_tst_results.append(tstresult)
if not grand_iter%20:
print 'epoch %4d' %trainer.totalepochs, 'train error %5.2f%%' %trnresult, \
'test error %5.2f%%' %tstresult
inputs = list(trndata['input'])
random.shuffle(inputs)
for inpt in trndata['input']:
nn.activate(inpt)
for minor_iter in range(hiddenAstrocyteLayer.astrocyte_processing_iters):
hiddenAstrocyteLayer.update()
outputAstrocyteLayer.update()
hiddenAstrocyteLayer.reset()
outputAstrocyteLayer.reset()
all_trn_results.append(iter_trn_results)
all_tst_results.append(iter_tst_results)
plotResults(all_trn_results)
plotResults(all_tst_results)
plt.show()
def train(args):
inputs, ys, gc = args
row_length = len(inputs[0])
d = ds.ClassificationDataSet(
row_length, nb_classes=2, class_labels=['Poisonous',
'Edible'])
d.setField('input', inputs)
d.setField('target', ys)
test, train = d.splitWithProportion(.25)
test._convertToOneOfMany()
train._convertToOneOfMany()
hidden = row_length // 2
print "indim:", train.indim
net = buildNetwork(train.indim,
hidden,
train.outdim,
outclass=SoftmaxLayer)
trainer = BackpropTrainer(net,
dataset=train,
momentum=0.0,
learningrate=0.1,
verbose=True,
weightdecay=0.0)
for i in xrange(20):
trainer.trainEpochs(1)
trnresult = percentError(trainer.testOnClassData(),
train['class'])
tstresult = percentError(
trainer.testOnClassData(dataset=test),
test['class'])
print "epoch: %4d" % trainer.totalepochs, \
" train error: %5.2f%%" % trnresult, \
" test error: %5.2f%%" % tstresult
return net, gc
def train(self):
#self.init_iri()
self.init_image()
self.ds = ClassificationDataSet(self.IN, 1, nb_classes=128)
#classifier.init_image()
self.load_data()
print "Number of trianing patterns: ", len(self.trndata)
print "Input and output dimensions: ", self.trndata.indim, self.trndata.outdim
print "First sample (input, target, class):"
print self.trndata['input'][0], self.trndata['target'][0], self.trndata['class'][0]
print self.trndata.indim, self.trndata.outdim
self.net = buildNetwork(self.trndata.indim, 7, self.trndata.outdim)
trainer = BackpropTrainer(self.net, dataset=self.trndata, momentum=0.1, verbose=True, weightdecay=0.01)
"""
for i in range(200):
trainer.trainEpochs(1)
trnresult = percentError(trainer.testOnClassData(), self.trndata['class'])
tstresult = percentError(trainer.testOnClassData(dataset = self.tstdata), self.tstdata["class"])
print "epch: %4d" % trainer.totalepochs, \
" train error: %5.2f%%" % trnresult, \
" test error: %5.2f%%" % tstresult
"""
trainer.trainUntilConvergence()
trnresult = percentError(trainer.testOnClassData(), self.trndata['class'])
tstresult = percentError(trainer.testOnClassData(dataset = self.tstdata), self.tstdata["class"])
print "epch: %4d" % trainer.totalepochs, \
" train error: %5.2f%%" % trnresult, \
" test error: %5.2f%%" % tstresult
def trainNetwork(inData, numOfSamples, numOfPoints, epochs):
# Build the dataset
alldata = createRGBdataSet(inData, numOfSamples, numOfPoints)
# Split into test and training data
trndata, tstdata = splitData(alldata)
# Report stats
print "Number of training patterns: ", len(trndata)
print "Input and output dimensions: ", trndata.indim, trndata.outdim
print "First sample (input, target, class):"
print trndata['input'][0], trndata['target'][0], trndata['class'][0]
# Build and train the network
fnn = buildNetwork( trndata.indim, 256, trndata.outdim, outclass=SoftmaxLayer )
trainer = BackpropTrainer( fnn, dataset=trndata, momentum=0.001, verbose=True, weightdecay=0.001)
#trainer.trainEpochs( epochs )
trainer.trainUntilConvergence(maxEpochs=epochs)
# Report results
trnresult = percentError( trainer.testOnClassData(), trndata['class'] )
tstresult = percentError( trainer.testOnClassData( dataset=tstdata ), tstdata['class'] )
print "epoch: %4d" % trainer.totalepochs, \
" train error: %5.2f%%" % trnresult, \
" test error: %5.2f%%" % tstresult
# Report results of final network
checkNeuralNet(trainer, alldata, numOfSamples)
return fnn
def train(self, inputData, verbose=True):
# Set of data to classify:
# - IMG_SIZE input dimensions per data point
# - 1 dimensional output
# - 4 clusters of classification
all_faces = ClassificationDataSet(IMG_SIZE, 1, nb_classes=4)
for entry in inputData:
(emotion, data) = entry
all_faces.addSample(data, [emotion])
# Generate a test and a train set from our data
test_faces, train_faces = all_faces.splitWithProportion(0.25)
# Hack to convert a 1-dimensional output into 4 output neurons
test_faces._convertToOneOfMany()
train_faces._convertToOneOfMany()
# Set up the actual network. These are the tunable params
self.fnn = buildNetwork(
train_faces.indim,
20,
train_faces.outdim,
outclass=SoftmaxLayer
)
# Set up the network trainer. Also nice tunable params
trainer = BackpropTrainer(
self.fnn,
dataset=train_faces,
momentum=0.1,
verbose=False,
weightdecay=0.01
)
tabledata = []
# Train this bitch.
if verbose:
# Report after every epoch if verbose
for i in range(EPOCHS):
trainer.trainEpochs(1)
trnresult = percentError( trainer.testOnClassData(),
train_faces['class'] )
tstresult = percentError( trainer.testOnClassData(
dataset=test_faces ), test_faces['class'] )
tabledata.append((trainer.totalepochs,trnresult,tstresult))
else:
trainer.trainEpochs(EPOCHS)
if verbose:
print "Epoch\tTrain Error\tTest Error"
for line in tabledata:
print "%4d\t" % line[0], \
"%5.2f%%\t\t" % line[1], \
"%5.2f%%" % line[2]
def nn_classify():
# train_X,Y = load_svmlight_file('data/train_metrix')
# rows = pd.read_csv('data/log_test2.csv',index_col=0).sort_index().index.unique()
# train_X = pd.read_csv('data/train_tfidf.csv',index_col=0)
# test_X = pd.read_csv('data/test_tfidf.csv',index_col=0)
# select = SelectPercentile(f_classif, percentile=50)
# select.fit(train_X,Y)
# train_X = select.transform(train_X)
# test_X = select.transform(test_X)
# print 'dump train...'
# dump_svmlight_file(train_X,Y,'data/train_last')
# test_Y = [0]*(test_X.shape[0])
# print 'dump test...'
# dump_svmlight_file(test_X,test_Y,'data/test_last')
train_X,Y = load_svmlight_file('data/train_last')
test_X,test_Y = load_svmlight_file('data/test_last')
train_X = train_X.toarray()
test_X = test_X.toarray()
Y = [int(y)-1 for y in Y]
print 'Y:',len(Y)
rows = pd.read_csv('data/log_test2.csv',index_col=0).sort_index().index.unique()
train_n = train_X.shape[0]
m = train_X.shape[1]
test_n = test_X.shape[0]
print train_n,m,#test_n
train_data = ClassificationDataSet(m,1,nb_classes=12)
test_data = ClassificationDataSet(m,1,nb_classes=12)
# test_data = ClassificationDataSet(test_n,m,nb_classes=12)
for i in range(train_n):
train_data.addSample(np.ravel(train_X[i]),Y[i])
for i in range(test_n):
test_data.addSample(test_X[i],Y[i])
trndata = train_data
# tstdata = train_data
trndata._convertToOneOfMany()
# tstdata._convertToOneOfMany()
test_data._convertToOneOfMany()
# 先用训练集训练出所有的分类器
print 'train classify...'
fnn = buildNetwork( trndata.indim, 400 , trndata.outdim, outclass=SoftmaxLayer )
trainer = BackpropTrainer( fnn, dataset=trndata, momentum=0.1, learningrate=0.01 , verbose=True, weightdecay=0.01)
trainer.trainEpochs(3)
# print 'Percent Error on Test dataset: ' , percentError( trainer.testOnClassData (
# dataset=tstdata )
# , )
print 'end train classify'
pre_y = trainer.testOnClassData(dataset=trndata)
print metrics.classification_report(Y,pre_y)
pre_y = trainer.testOnClassData(dataset=test_data)
print 'write result...'
print 'before:',pre_y[:100]
pre_y = [int(y)+1 for y in pre_y]
print 'after:',pre_y[:100]
DataFrame(pre_y,index=rows).to_csv('data/info_test2.csv', header=False)
print 'end...'
def runNeuralLearningCurveSimulation(dataTrain, dataTest, train_tfidf, test_tfidf, outFile):
print 'running neural learning curve'
outFile.write('-------------------------------------\n')
outFile.write('train==> %d, %d \n'%(train_tfidf.shape[0],train_tfidf.shape[1]))
outFile.write('test==> %d, %d \n'%(test_tfidf.shape[0],test_tfidf.shape[1]))
trainDS = getDataSetFromTfidf(train_tfidf, dataTrain.target)
testDS = getDataSetFromTfidf(test_tfidf, dataTest.target)
print "Number of training patterns: ", len(trainDS)
print "Input and output dimensions: ", trainDS.indim, trainDS.outdim
print "First sample (input, target, class):"
print len(trainDS['input'][0]), trainDS['target'][0], trainDS['class'][0]
'''
with SimpleTimer('time to train', outFile):
net = buildNetwork(trainDS.indim, trainDS.indim/2, trainDS.indim/4, trainDS.indim/8, trainDS.indim/16, 2, hiddenclass=TanhLayer, outclass=SoftmaxLayer)
trainer = BackpropTrainer( net, dataset=trainDS, momentum=0.1, verbose=True, weightdecay=0.01, batchlearning=True)
'''
net = RecurrentNetwork()
net.addInputModule(LinearLayer(trainDS.indim, name='in'))
net.addModule(SigmoidLayer(trainDS.indim/2, name='hidden'))
net.addModule(SigmoidLayer(trainDS.indim/4, name='hidden2'))
net.addOutputModule(SoftmaxLayer(2, name='out'))
net.addConnection(FullConnection(net['in'], net['hidden'], name='c1'))
net.addConnection(FullConnection(net['hidden'], net['out'], name='c2'))
net.addRecurrentConnection(FullConnection(net['hidden'], net['hidden'], name='c3'))
net.addRecurrentConnection(FullConnection(net['hidden2'], net['hidden'], name='c4'))
net.sortModules()
trainer = BackpropTrainer( net, dataset=trainDS, momentum=0.01, verbose=True, weightdecay=0.01)
outFile.write('%s \n' % (net.__str__()))
epochs = 200
with SimpleTimer('time to train %d epochs' % epochs, outFile):
for i in range(epochs):
trainer.trainEpochs(1)
trnresult = percentError( trainer.testOnClassData(),
trainDS['class'] )
tstresult = percentError( trainer.testOnClassData(
dataset=testDS ), testDS['class'] )
print "epoch: %4d" % trainer.totalepochs, \
" train error: %5.2f%%" % trnresult, \
" test error: %5.2f%%" % tstresult
outFile.write('%5.2f , %5.2f \n' % (100.0-trnresult, 100.0-tstresult))
predicted = trainer.testOnClassData(dataset=testDS)
results = predicted == testDS['class'].flatten()
wrong = []
for i in range(len(results)):
if not results[i]:
wrong.append(i)
print 'classifier got these wrong:'
for i in wrong[:10]:
print dataTest.data[i], dataTest.target[i]
outFile.write('%s %d \n' % (dataTest.data[i], dataTest.target[i]))
def __trainNetwork(self,trndata,tstdata):
self.fnn = buildNetwork( trndata.indim, 2, trndata.outdim, outclass=SigmoidLayer )
trainer = BackpropTrainer( self.fnn, dataset=trndata, momentum=0.1, verbose=True, weightdecay=0.01,learningrate=0.01)
for i in range(self.iterations):
trainer.trainEpochs( 1 )
trnresult = percentError( trainer.testOnClassData(),trndata['class'] )
tstresult = percentError( trainer.testOnClassData(dataset=tstdata ), tstdata['class'] )
print "epoch: %4d" % trainer.totalepochs, " train error: %5.2f%%" % trnresult," test error: %5.2f%%" % tstresult
def basicneuralnetwork(number_of_hidden_nodes, weightdecay, layers, alldata):
'''
This is a dataset
first argument is the dimension of the input
second argument is dimension of the output
'''
nr_of_iterations = 10
# Construct neural network
print "Constructing network"
print " (number_of_hidden_nodes : %s, weight decay : %s, layers: %s)" % (number_of_hidden_nodes, weightdecay, layers)
train_results = []
test_results = []
neural_networks = []
for i in xrange(1,nr_of_iterations+1):
print "Iteration %d" % i
# Prepare the data
tstdata, trndata = alldata.splitWithProportion( 0.15 )
trndata._convertToOneOfMany()
tstdata._convertToOneOfMany()
# construct a neural network
fnn = construct_neural_network(number_of_hidden_nodes, layers, trndata.indim, trndata.outdim)
trainer = BackpropTrainer( fnn, dataset=trndata, momentum=0.1, verbose=False, weightdecay=weightdecay)
#early stopping validation set = 0.25
trainer.trainUntilConvergence(continueEpochs=5)
train_results.append(percentError( trainer.testOnClassData(), trndata['class']))
test_results.append(percentError( trainer.testOnClassData(dataset=tstdata ), tstdata['class']))
neural_networks.append(fnn)
global activation_samples
for sample in activation_samples:
print "The activation sample: %s"%sample
print "The output :%s"%fnn.activate(sample)
print "Weight:%s , %s"%(fnn['in'].outputbuffer[fnn['in'].offset],fnn['hidden0'].outputbuffer[fnn['hidden0'].offset])
log_debug(trndata.indim,number_of_hidden_nodes, weightdecay,train_results[-1],test_results[-1])
# Compute means
mean_train_error = sum(train_results)/len(train_results)
mean_test_error = sum(test_results)/len(test_results)
# Compute optimal network configuration
optimal_test_error = min(test_results)
optimal_index = test_results.index(optimal_test_error)
# Save the optimal configuration to the file system
import os
neuralnetwork = os.path.join('networks', 'neuralHiddenNode%sdecay%s'%(number_of_hidden_nodes, weightdecay))
fileObject = open(neuralnetwork, 'w')
pickle.dump(neural_networks[optimal_index], fileObject)
fileObject.close()
return (mean_train_error, mean_test_error)
def trainNN(data: list, targets: list, seed):
"""
Trains a neural network
"""
X_tweet_counts = count_vect.fit_transform(data)
# Compute term frequencies and store in X_train_tf
# Compute tfidf feature values and store in X_train_tfidf
X_train_tfidf = tfidf_transformer.fit_transform(X_tweet_counts)
arr = X_train_tfidf.toarray()
trainingdata = arr[:int(.75 * len(arr))]
testdata = arr[int(.75 * len(arr)):]
trainingtargets = targets[:int(.75 * len(targets))]
testtargets = targets[int(.75 * len(targets)):]
trainingds = ClassificationDataSet(len(arr[0]), 1, nb_classes=2)
testds = ClassificationDataSet(len(arr[0]), 1, nb_classes=2)
for index, data in enumerate(trainingdata):
trainingds.addSample(data, trainingtargets[index])
for index, data in enumerate(testdata):
testds.addSample(data, testtargets[index])
trainingds._convertToOneOfMany()
testds._convertToOneOfMany()
net = buildNetwork(trainingds.indim, 10, 10, 10, trainingds.outdim, outclass=SoftmaxLayer)
trainer = BackpropTrainer(net, dataset=trainingds, learningrate=.65, momentum=.1)
besttrain = 99.9
besttest = 99.9
bestresults = []
bestclass = []
for i in range(20):
trainer.trainEpochs(1)
trainresult = percentError(trainer.testOnClassData(), trainingds['class'])
teststuff = trainer.testOnClassData(dataset=testds)
testresult = percentError(teststuff, testds['class'])
if testresult < besttest:
besttest = testresult
besttrain = trainresult
bestresults = teststuff
bestclass = testds['class']
print("epoch: %2d" % trainer.totalepochs)
print("train error: %2.2f%%" % trainresult)
print("test error: %2.2f%%" % testresult)
print("Best test error accuracy: {:.2f}%".format(besttest))
print("Best test error f1 score: {:.4f}%".format(f1_score(bestclass, bestresults, average='macro')))
print("Confusion Matrix:")
print(confusion_matrix(bestclass, bestresults))
return besttest
def main():
means = [(-1,0),(2,4),(3,1)]
cov = [diag([1,1]), diag([0.5,1.2]), diag([1.5,0.7])]
alldata = ClassificationDataSet(2, 1, nb_classes=3)
for n in xrange(400):
for klass in range(3):
input = multivariate_normal(means[klass],cov[klass])
alldata.addSample(input, [klass])
tstdata, trndata = alldata.splitWithProportion( 0.25 )
trndata._convertToOneOfMany( )
tstdata._convertToOneOfMany( )
print "Number of training patterns: ", len(trndata)
print "Input and output dimensions: ", trndata.indim, trndata.outdim
print "First sample (input, target, class):"
print trndata['input'][0], trndata['target'][0], trndata['class'][0]
fnn = buildNetwork( trndata.indim, 5, trndata.outdim, outclass=SoftmaxLayer )
trainer = BackpropTrainer( fnn, dataset=trndata, momentum=0.1, verbose=True, weightdecay=0.01)
ticks = arange(-3.,6.,0.2)
X, Y = meshgrid(ticks, ticks)
# need column vectors in dataset, not arrays
griddata = ClassificationDataSet(2,1, nb_classes=3)
for i in xrange(X.size):
griddata.addSample([X.ravel()[i],Y.ravel()[i]], [0])
griddata._convertToOneOfMany() # this is still needed to make the fnn feel comfy
for i in range(20):
trainer.trainEpochs(1)
trnresult = percentError( trainer.testOnClassData(),
trndata['class'] )
tstresult = percentError( trainer.testOnClassData(
dataset=tstdata ), tstdata['class'] )
print "epoch: %4d" % trainer.totalepochs, \
" train error: %5.2f%%" % trnresult, \
" test error: %5.2f%%" % tstresult
out = fnn.activateOnDataset(griddata)
out = out.argmax(axis=1) # the highest output activation gives the class
out = out.reshape(X.shape)
figure(1)
ioff() # interactive graphics off
clf() # clear the plot
hold(True) # overplot on
for c in [0,1,2]:
here, _ = where(tstdata['class']==c)
plot(tstdata['input'][here,0],tstdata['input'][here,1],'o')
if out.max()!=out.min(): # safety check against flat field
contourf(X, Y, out) # plot the contour
ion() # interactive graphics on
draw() # update the plot
ioff()
show()
def EvaluateArtificialNeuralNetwork(training_data, Input_features, Output_feature, NUMBER_CLASSES, HIDDEN_NEURONS, NUMBER_LAYERS, dataset_name, ParameterVal):
X = training_data[Input_features]
Y = training_data[Output_feature]
ds = ClassificationDataSet(X.shape[1], nb_classes=NUMBER_CLASSES)
for k in xrange(len(X)):
ds.addSample((X.ix[k,:]), Y.ix[k,:])
tstdata_temp, trndata_temp = ds.splitWithProportion(.25)
tstdata = ClassificationDataSet(X.shape[1], nb_classes=NUMBER_CLASSES)
for n in xrange(0, tstdata_temp.getLength()):
tstdata.addSample( tstdata_temp.getSample(n)[0], tstdata_temp.getSample(n)[1] )
trndata = ClassificationDataSet(X.shape[1], nb_classes=NUMBER_CLASSES)
for n in xrange(0, trndata_temp.getLength()):
trndata.addSample( trndata_temp.getSample(n)[0], trndata_temp.getSample(n)[1] )
if NUMBER_CLASSES > 1:
trndata._convertToOneOfMany( )
tstdata._convertToOneOfMany( )
'''*****Actual computation with one layer and HIDDEN_NEURONS number of neurons********'''
fnn = buildNetwork( trndata.indim, HIDDEN_NEURONS , trndata.outdim, outclass=SoftmaxLayer )
trainer = BackpropTrainer( fnn, dataset=trndata, momentum=0.1, verbose=False, weightdecay=0.01)
trainer.trainUntilConvergence(maxEpochs=3)
trnresult = percentError( trainer.testOnClassData(), trndata['class'] )
tstresult = percentError( trainer.testOnClassData(dataset=tstdata ), tstdata['class'] )
print ("Accuracy with Artificial Neural Network: epoch: " + str(trainer.totalepochs) + " TrainingSet:" + str(1-trnresult/100) + " TestSet:" + str(1-tstresult/100))
'''****** Graphical Representation*****'''
'''tot_hidden_tests, X_train, X_test, Y_train, Y_test, training_error, test_error = InitiateErrorCalcData(ParameterVal, training_data[Input_features], training_data[Output_feature])
for hidden_unit in tot_hidden_tests:
print ("Computing hidden unit :" + str(hidden_unit))
model = buildNetwork( trndata.indim, hidden_unit , trndata.outdim, outclass=SoftmaxLayer )
temp_trainer = BackpropTrainer( model, dataset=trndata, momentum=0.1, verbose=True, weightdecay=0.01)
temp_trainer.trainUntilConvergence(maxEpochs=3)
training_error.append(MSE( temp_trainer.testOnClassData(), trndata['class'] ))
test_error.append(MSE( temp_trainer.testOnClassData(dataset=tstdata ), tstdata['class'] ))
PlotErrors(tot_hidden_tests, training_error, test_error, dataset_name, "Number of Hidden Units for single layer ANN", "MSE")'''
'''*****Graphical representation with multiple layers and HIDDEN_NEURONS number of neurons********'''
'''ffn = FeedForwardNetwork()
def perceptron(hidden_neurons=5, weightdecay=0.01, momentum=0.1):
INPUT_FEATURES = 41
CLASSES = 10
HIDDEN_NEURONS = hidden_neurons
WEIGHTDECAY = weightdecay
MOMENTUM = momentum
# Generate the labeled set
g = load_data()
#g = generate_data2()
alldata = g['d']
minX, maxX, minY, maxY = g['minX'], g['maxX'], g['minY'], g['maxY']
# Split data into test and training dataset
tstdata, trndata = alldata.splitWithProportion(0.25)
trndata._convertToOneOfMany() # This is necessary, but I don't know why
tstdata._convertToOneOfMany() # http://stackoverflow.com/q/8154674/562769
print("Number of training patterns: %i" % len(trndata))
print("Input and output dimensions: %i, %i" % (trndata.indim,
trndata.outdim))
print("Hidden neurons: %i" % HIDDEN_NEURONS)
print("First sample (input, target, class):")
#print(trndata['input'][0], trndata['target'][0], trndata['class'])
fnn = buildNetwork(trndata.indim, HIDDEN_NEURONS, trndata.outdim,bias=True)
#outclass=SoftmaxLayer)
trainer = BackpropTrainer(fnn, dataset=trndata, momentum=MOMENTUM,
verbose=True, weightdecay=WEIGHTDECAY)
# Visualization
for i in range(20):
trainer.trainEpochs(10)
trnresult = percentError(trainer.testOnClassData(),
trndata['class'])
tstresult = percentError(trainer.testOnClassData(
dataset=tstdata), tstdata['class'])
print("epoch: %4d" % trainer.totalepochs,
" train error: %5.2f%%" % trnresult,
" test error: %5.2f%%" % tstresult)
test = [0]*41
test[0],test[3],test[40],test[39],test[38],test[37],test[36] = 1,1,1,1,1,1,1
print fnn.activate(test)
return fnn
"""
def main():
trndata, tstdata = createDS()
for repeat in xrange(repeats):
iter_trn_results = []
iter_tst_results = []
nn = createNN(4, 6, 3)
nn.randomize()
print nn.params
hiddenAstrocyteLayer, outputAstrocyteLayer = associateAstrocyteLayers(nn)
trainer = BackpropTrainer(nn, dataset=trndata, learningrate=0.01,
momentum=0.1, verbose=False, weightdecay=0.0)
for grand_iter in xrange(iterations):
if grand_iter == 0:
trainer.train()
if grand_iter > iterations/3 and grand_iter < iterations/3:
inputs = trndata['input'][:]
random.shuffle(inputs)
for inpt in trndata['input']:
nn.activate(inpt)
for minor_iter in range(hiddenAstrocyteLayer.astrocyte_processing_iters):
hiddenAstrocyteLayer.update()
outputAstrocyteLayer.update()
hiddenAstrocyteLayer.reset()
outputAstrocyteLayer.reset()
trainer.train()
trnresult = percentError(trainer.testOnClassData(), trndata['class'])
iter_trn_results.append(trnresult)
tstresult = percentError(trainer.testOnClassData(dataset=tstdata), tstdata['class'])
iter_tst_results.append(tstresult)
if not grand_iter%100:
print 'epoch %4d' %trainer.totalepochs, 'train error %5.2f%%' %trnresult, \
'test error %5.2f%%' %tstresult
all_trn_results.append(iter_trn_results)
all_tst_results.append(iter_tst_results)
path = '/home/david/Dropbox/programming/python/ANN (Case Conflict 1)/pybrain/'
f = plt.figure(figsize=(10,5))
plotErrorBar(all_trn_results)
plotErrorBar(all_tst_results)
plt.legend(('Training', 'Test'))
f.savefig(path+'angn_bpThenAstro_error_bar.svg', format='svg')
f = plt.figure(figsize=(10,5))
side_text = 'Trials='+str(repeats)+'\n\nFinal\ntrain\nerror\n'
plotNoErrorBar(all_trn_results, label=side_text, xytxt=(1.02, 0.7))
plotNoErrorBar(all_tst_results, label='Final\ntest\nerror\n', xytxt=(1.02, 0.4))
plt.legend(('Training', 'Test'))
f.savefig(path+'angn_bpThenAstro_no_error_bar.svg', format='svg')
plt.close()
def mlpClassifier(X,y,train_indices, test_indices, mom=0.1,weightd=0.01, epo=5):
X_train, y_train, X_test, y_test = X[train_indices],y[train_indices], X[test_indices], y[test_indices]
#Converting the data into a dataset which is easily understood by PyBrain.
tstdata = ClassificationDataSet(X.shape[1],target=1,nb_classes=8)
trndata = ClassificationDataSet(X.shape[1],target=1,nb_classes=8)
# print "shape of X_train & y_train: " + str(X_train.shape) + str(y_train.shape)
for i in range(y_train.shape[0]):
trndata.addSample(X_train[i,:], y_train[i])
for i in range(y_test.shape[0]):
tstdata.addSample(X_test[i,:], y_test[i])
trndata._convertToOneOfMany()
tstdata._convertToOneOfMany()
#printing the specs of data
# print "Number of training patterns: ", len(trndata)
# print "Input and output dimensions: ", trndata.indim, trndata.outdim
# print "First sample (input, target, class):"
# print trndata['input'][0], trndata['target'][0], trndata['class'][0]
#The neural-network used
# print "Building Network..."
#input layer, hidden layer of size 10(very small), output layer
ANNc = FeedForwardNetwork()
inLayer = LinearLayer(trndata.indim, name="ip")
hLayer1 = TanhLayer(100, name = "h1")
hLayer2 = SigmoidLayer(100, name = "h2")
outLayer = SoftmaxLayer(trndata.outdim, name = "op")
ANNc.addInputModule(inLayer)
ANNc.addModule(hLayer1)
ANNc.addModule(hLayer2)
ANNc.addOutputModule(outLayer)
ip_to_h1 = FullConnection(inLayer, hLayer1, name = "ip->h1")
h1_to_h2 = FullConnection(hLayer1, hLayer2, name = "h1->h2")
h2_to_op = FullConnection(hLayer2, outLayer, name = "h2->op")
ANNc.addConnection(ip_to_h1)
ANNc.addConnection(h1_to_h2)
ANNc.addConnection(h2_to_op)
ANNc.sortModules()
# print "Done. Training the network."
#The trainer used, in our case Back-propagation trainer
trainer = BackpropTrainer( ANNc, dataset=trndata, momentum=mom, verbose=True, weightdecay=weightd)
trainer.trainEpochs( epo )
#The error
trnresult = percentError( trainer.testOnClassData(dataset=trndata), trndata['class'] )
tstresult = percentError( trainer.testOnClassData(dataset=tstdata ), tstdata['class'] )
# print "Done."
return ANNc, trainer.totalepochs, (100 - trnresult), (100 - tstresult)
def train_network(trndata, tstdata, num_epochs=1000, log_frequency=100):
fnn = buildNetwork( trndata.indim, 100, trndata.outdim, outclass=SoftmaxLayer )
trainer = BackpropTrainer( fnn, dataset=trndata, momentum=0.01, verbose=True, weightdecay=0)
for i in xrange(num_epochs):
trainer.trainEpochs( 1 )
trnresult = percentError( trainer.testOnClassData(),
trndata['class'] )
tstresult = percentError( trainer.testOnClassData(
dataset=tstdata ), tstdata['class'] )
if i % log_frequency == 0:
print "epoch: %4d" % trainer.totalepochs,\
" train error: %5.2f%%" % trnresult,\
" test error: %5.2f%%" % tstresult
def run(self, fold, X_train, y_train, X_test, y_test):
DS_train, DS_test = ClassificationData.convert_to_DS(
X_train,
y_train,
X_test,
y_test)
NHiddenUnits = self.__get_best_hu(DS_train)
fnn = buildNetwork(
DS_train.indim,
NHiddenUnits,
DS_train.outdim,
outclass=SoftmaxLayer,
bias=True)
trainer = BackpropTrainer(
fnn,
dataset=DS_train,
momentum=0.1,
verbose=False,
weightdecay=0.01)
trainer.trainEpochs(self.epochs)
tstresult = percentError(
trainer.testOnClassData(dataset=DS_test),
DS_test['class'])
print "NN fold: %4d" % fold, "; test error: %5.2f%%" % tstresult
return tstresult / 100.0
def run_nn_fold(training_data, test_data):
test_features, ignore, featureMap, labels, labelMap = fs.mutualinfo(training_data)
input_len = len(test_features[0])
num_classes = len(labelMap.keys())
train_ds = ClassificationDataSet(input_len, 1,nb_classes=num_classes)
for i in range(len(test_features)):
train_ds.addSample(tuple(test_features[i]), (labels[i]))
train_ds._convertToOneOfMany()
net = buildNetwork(train_ds.indim, 2, train_ds.outdim, bias=True, hiddenclass=TanhLayer, outclass=SoftmaxLayer)
trainer = BackpropTrainer(net, train_ds, verbose=True)
print "training until convergence..."
trainer.trainUntilConvergence(maxEpochs=100)
print "done. testing..."
test_ds = ClassificationDataSet(input_len, 1,nb_classes=num_classes)
labels = []
for tweetinfo in test_data:
featuresFound = tweetinfo["Features"]
label = tweetinfo["Answer"]
labels.append(label)
features = [0]*len(featureMap.keys())
for feat in featuresFound:
if feat in featureMap:
features[ featureMap[feat] ] = 1
test_ds.addSample(tuple(features), (labelMap[label]))
test_ds._convertToOneOfMany()
tstresult = percentError( trainer.testOnClassData(
dataset=test_ds ), test_ds['class'] )
print tstresult
def ann(training_filename , testing_filename,itr,epoch,model_type):
training_start_time = "The generation of data set and training started at :%s" % datetime.datetime.now()
training_dataset = np.genfromtxt(training_filename, skip_header=0,dtype="int", delimiter='\t' )
data = ClassificationDataSet(len(training_dataset[0])-1, 2, nb_classes=2)
for aSample in training_dataset:
data.addSample(aSample[0:len(aSample)-1],[aSample[len(aSample)-1]] );
#
data._convertToOneOfMany( )
fann = buildNetwork(314,2,outclass=SoftmaxLayer);
trainer = BackpropTrainer( fann, dataset=data, momentum=0.1, verbose=False, weightdecay=0.01)
counter = 0;
print training_start_time
while(counter < itr):
trainer.trainEpochs( epoch );
counter = counter + 1;
trnresult = percentError( trainer.testOnClassData(),data['class'] )
trained_result_log = "epoch: %4d" % trainer.totalepochs, \
" train error: %5.2f%%" % trnresult;
training_time_end = "The training and result logging ended at %s :" % datetime.datetime.now()
filename = working_dir + "\models\\"+model_type + ".obj"
save_trained_model(fann, filename)
log_file.write("\n" + training_start_time+"\n")
log_file.write(str(trained_result_log)+"\n")
log_file.write(training_time_end+"\n")
def run(self, ds_train, ds_test):
"""
This function both trains the ANN and evaluates the ANN using a specified training and testing set
Args:
:param ds_train (TweetClassificationDatasetFactory): the training dataset the neural network is trained with.
:param ds_test (TweetClassificationDatasetFactory): the test dataset evaluated.
:returns: error (float): the percent error of the test dataset, tested on the neural network.
"""
ds_train._convertToOneOfMany()
ds_test._convertToOneOfMany()
trainer = BackpropTrainer(
self.network,
dataset=ds_train,
momentum=0.1,
verbose=True,
weightdecay=0.01)
trainer.trainUntilConvergence(
dataset=ds_train,
maxEpochs=self.max_epochs,
continueEpochs=self.con_epochs)
result = trainer.testOnClassData(dataset=ds_test)
error = percentError(result, ds_test['class'])
return error
class Network():
network = None
trainer = None
hidden_layer = None
hidden_nodes = None
data = None
def __init__(self, data, n_hidden_nodes, layertype="Linear"):
self.hidden_layer = NH.layers[layertype]
self.data = data
train_dim_in = data.train_data.indim
train_dim_out = data.train_data.outdim
self.network = buildNetwork(train_dim_in, n_hidden_nodes, train_dim_out, outclass=self.hidden_layer)
self.hidden_nodes = n_hidden_nodes
def init_backprop_trainer(self, b_momentum=0.1, b_learningrate=0.01, b_verbose=True, b_weightdecay=0.1):
train_in = self.data.train_data.indim
train_out = self.data.train_data.outdim
self.trainer = BackpropTrainer(self.network, dataset=self.data.train_data, \
momentum=b_momentum, learningrate=b_learningrate, verbose=b_verbose, \
weightdecay=b_weightdecay)
def run_network(self, epoch):
NetworkWriter.writeToFile(self.network, "test.xml")
self.trainer.trainEpochs(epoch)
error = percentError(self.trainer.testOnClassData(dataset=self.data.test_data), \
self.data.test_data['class'])
return error
"""
def fnn_classify(ds_train, ds_test):
"""Train neural network without bagging"""
net = buildNetwork(24, 18, 16, 8, hiddenclass=TanhLayer, outclass=SoftmaxLayer) # define neural net
trainer = BackpropTrainer(net, dataset=ds_train, learningrate=0.01, momentum=0.1, verbose=True, weightdecay=0.01)
trainer.trainEpochs(20) # train
test_result = percentError(trainer.testOnClassData(
dataset=ds_test), ds_test['class'])
return 100-test_result
index = numbers.index(letter)
y[index] = 1
testing.addSample(segement.flatten(), y)
print('\n')
# 创建一个 输入层为400个神经元，隐藏测为100个，输出层为10个的神经网络结构，使用BP神经网络算法（反向传播）
net = buildNetwork(400, 5, len(numbers), bias=True)
trainer = BackpropTrainer(net, training, learningrate=0.01, weightdecay=0.01)
# 设置训练步数
trainer.trainEpochs(epochs=50)
# 保存模型
pickle.dump(trainer, open('number——tow_predictor.model', 'wb'), 0)
# 测试
predictions = trainer.testOnClassData(dataset=testing)
print("预测数字：")
for v in predictions:
print(numbers[v], end='')
print()
# 读取图片，，经过反相处理，重定尺寸处理，模式转换为L
image = Image.open('1.png')
plt.imshow(image)
image = ImageOps.invert(image)
image = image.resize((15, 20))
image = np.array(image.convert('L'))
image = image / image.max() # 归一
image = segement_image(image)[0] # 分割
ds.addSample(x.iloc[k], np.array(y[k]))
tstdata, trndata = ds.splitWithProportion(0.20)
trndata._convertToOneOfMany()
tstdata._convertToOneOfMany()
input_size = len(x.columns)
target_size = 1
hidden_size = 5
fnn = None
if os.path.isfile('fnn.xml'):
fnn = NetworkReader.readFrom('fnn.xml')
else:
fnn = buildNetwork(trndata.indim,
hidden_size,
trndata.outdim,
outclass=SoftmaxLayer)
trainer = BackpropTrainer(fnn,
dataset=trndata,
momentum=0.05,
learningrate=0.1,
verbose=False,
weightdecay=0.01)
trainer.trainUntilConvergence(verbose=False,
validationProportion=0.15,
maxEpochs=100,
continueEpochs=10)
NetworkWriter.writeToFile(fnn, 'oliv.xml')
predictions = trainer.testOnClassData(dataset=tstdata)
y_test, y_pred = tstdata['class'], predictions
# Saving model to disk
pickle.dump(trainer, open('neural_network.pkl', 'wb'), protocol=2)
# convert a supervised dataset to a classification dataset
def _convert_supervised_to_classification(supervised_dataset,classes):
classification_dataset = ClassificationDataSet(supervised_dataset.indim,supervised_dataset.outdim,classes)
for n in xrange(0, supervised_dataset.getLength()):
classification_dataset.addSample(supervised_dataset.getSample(n)[0], supervised_dataset.getSample(n)[1])
return classification_dataset
olivetti = datasets.fetch_olivetti_faces()
X, y = olivetti.data, olivetti.target
ds = ClassificationDataSet(4096, 1 , nb_classes=40)
for k in xrange(len(X)):
ds.addSample(np.ravel(X[k]),y[k])
tstdata, trndata = ds.splitWithProportion( 0.25 )
tstdata = _convert_supervised_to_classification(tstdata,40)
trndata = _convert_supervised_to_classification(trndata,40)
trndata._convertToOneOfMany( )
tstdata._convertToOneOfMany( )
#fnn = buildNetwork( trndata.indim, 64 , trndata.outdim, outclass=SoftmaxLayer )
if os.path.isfile('oliv.xml'):
fnn = NetworkReader.readFrom('oliv.xml')
trainer = BackpropTrainer( fnn, dataset=trndata, momentum=0.1, learningrate=0.01 , verbose=True, weightdecay=0.01)
trainer.trainEpochs (50)
else:
fnn = buildNetwork( trndata.indim, 64 , trndata.outdim, outclass=SoftmaxLayer )
trainer = BackpropTrainer( fnn, dataset=trndata, momentum=0.1, learningrate=0.01 , verbose=True, weightdecay=0.01)
trainer.trainEpochs (50)
NetworkWriter.writeToFile(fnn, 'oliv.xml')
print 'Percent Error on Test dataset: ' , percentError( trainer.testOnClassData (dataset=tstdata ), tstdata['class'] )
trainer = BackpropTrainer(fnn,
dataset=trndata,
momentum=0.1,
lrdecay=1.0,
verbose=True,
weightdecay=0.001)
# <codecell>
for i in range(0, 3):
trainer.trainEpochs(1)
# <codecell>
trainer.testOnClassData()[:20], list(targets_train[:20])
# <codecell>
trnresult = percentError(trainer.testOnClassData(), trndata['class'])
tstresult = percentError(trainer.testOnClassData(dataset=tstdata),
tstdata['class'])
print "epoch: %4d" % trainer.totalepochs, \
" train error: %5.2f%%" % trnresult, \
" test error: %5.2f%%" % tstresult
# <codecell>
# Now predict the value of the digit on the second half:
targets_train_validation = classifier.predict(
zoomed_features_train[n_training:])
ds._convertToOneOfMany()
ts._convertToOneOfMany()
n = buildNetwork(ds.indim,
14,
14,
ds.outdim,
recurrent=True,
outclass=SoftmaxLayer)
t = BackpropTrainer(n,
dataset=ds,
learningrate=0.01,
momentum=0.5,
verbose=True)
t.trainEpochs(5)
#t.trainUntilConvergence(dataset=ds, maxEpochs=10000, verbose=True)
trnresult = percentError(t.testOnClassData(), ds['class'])
testresult = percentError(t.testOnClassData(), ts['class'])
print "epoch: %4d" % t.totalepochs, " train error: %5.2f%%" % trnresult, " test error: %5.2f%%" % testresult
#guesses = []
#
#def one_or_zero(array):
# return array[1]>.5
#
#for line in test_final:
# guesses.append(one_or_zero(n.activate(line)))
# final_guess = pd.DataFrame(guesses,columns=['requester_received_pizza'],dtype=int).join(test_ids)
# final_guess.set_index('request_id', inplace=True)
# final_guess.to_csv('/desktop/submission.csv')
class NeuralNetwork:
def __init__(self,
data,
learning_rate=0.1,
momentum=0.1,
n_hidden_units=5):
self.features = data['features']
self.weights = data['weights']
labels = data['labels']
self.labels = np.array([1 if l == 's' else 0 for l in labels])
self.learning_rate = learning_rate
self.momentum = momentum
self.n_hidden_units = n_hidden_units
self._prepare_data()
self._build_network(n_hidden_units)
def _prepare_data(self):
self.dataset = split_dataset(self.features, self.weights, self.labels)
classes = set(self.labels)
def training_set():
ds = ClassificationDataSet(
self.dataset['training']['features'].shape[1],
1,
nb_classes=len(classes))
for i in range(self.dataset['training']['features'].shape[0]):
ds.addSample(self.dataset['training']['features'][i],
self.dataset['training']['labels'][i])
return ds
def test_set():
ds = ClassificationDataSet(self.features.shape[1],
1,
nb_classes=len(classes))
for i in range(self.dataset['test']['features'].shape[0]):
ds.addSample(self.dataset['test']['features'][i],
self.dataset['test']['labels'][i])
return ds
self.trndata = training_set()
self.tstdata = test_set()
self.tstdata._convertToOneOfMany()
self.trndata._convertToOneOfMany()
def _build_network(self, n_hidden_units):
self.fnn = buildNetwork(self.trndata.indim,
n_hidden_units,
self.trndata.outdim,
outclass=SoftmaxLayer)
def _create_trainer(self, learning_rate, momentum):
self.trainer = BackpropTrainer(self.fnn,
dataset=self.trndata,
momentum=momentum,
verbose=False,
weightdecay=0.01,
learningrate=learning_rate)
def train(self, train_epoch=5):
self._create_trainer(self.learning_rate, self.momentum)
self.trainer.trainEpochs(train_epoch)
def learn_weights(self, max_evaluations, algoritm):
alg = algoritm(self.trndata.evaluateModuleMSE,
self.fnn,
verbose=False,
minimize=True,
maxEvaluations=max_evaluations)
for i in range(max_evaluations):
self.fnn = alg.learn(0)[0]
def predict(self, dataset=None):
if dataset is None:
dataset = self.tstdata
out = self.fnn.activateOnDataset(dataset)
out = out.argmax(axis=1)
return out
def estimate_error(self):
trnerror = percentError(
self.trainer.testOnClassData(dataset=self.trndata),
self.trndata['class'])
tsterror = percentError(
self.trainer.testOnClassData(dataset=self.tstdata),
self.tstdata['class'])
return self.trainer.totalepochs, trnerror, tsterror
def train_accuracy(self):
return accuracy_score(y_pred=self.predict(self.trndata),
y_true=self.trndata['class'])
def test_accuracy(self):
return accuracy_score(y_pred=self.predict(self.tstdata),
y_true=self.tstdata['class'])
momentum=0.1,
verbose=True,
weightdecay=0.01)
ticks = arange(-3., 6., 0.2)
X, Y = meshgrid(ticks, ticks)
# need column vectors in dataset, not arrays
griddata = ClassificationDataSet(2, 1, nb_classes=3)
for i in xrange(X.size):
griddata.addSample([X.ravel()[i], Y.ravel()[i]], [0])
griddata._convertToOneOfMany(
) # this is still needed to make the fnn feel comfy
for i in range(20):
trainer.trainEpochs(1)
trnresult = percentError(trainer.testOnClassData(), trndata['class'])
tstresult = percentError(trainer.testOnClassData(dataset=tstdata),
tstdata['class'])
print "epoch: %4d" % trainer.totalepochs, \
" train error: %5.2f%%" % trnresult, \
" test error: %5.2f%%" % tstresult
out = fnn.activateOnDataset(griddata)
out = out.argmax(axis=1) # the highest output activation gives the class
out = out.reshape(X.shape)
figure(1)
ioff() # interactive graphics off
clf() # clear the plot
hold(True) # overplot on
for c in [0, 1, 2]:
here, _ = where(tstdata['class'] == c)
def run(fold_object_path, number_of_fold, outpath, feature):
for i in range(number_of_fold):
test_fold_path = '{}{}.pickle'.format(fold_object_path, i)
# print test_fold_path
train_fold_path = []
for j in range(number_of_fold):
if j==i : continue
fold_path = '{}{}.pickle'.format(fold_object_path, j)
train_fold_path.append(fold_path)
# print train_fold_path
trndata = get_training_object(train_fold_path, feature)
# print trndata
tstdata = get_training_object([test_fold_path], feature)
# print tstdata
print "Number of training patterns: ", len(trndata)
print "Input and output dimensions: ", trndata.indim, trndata.outdim
print "First sample (input, target, class):"
fnn = buildNetwork(trndata.indim, 20, trndata.outdim, hiddenclass=TanhLayer, bias=False)
trainer = BackpropTrainer( fnn, dataset=trndata, momentum=0.1, weightdecay=0.01)
acc = 0.0
for i in range(50):
trainer.trainEpochs( 1 )
trnresult = percentError( trainer.testOnClassData(),
trndata['class'] )
tstresult = percentError( trainer.testOnClassData(
dataset=tstdata ), tstdata['class'] )
print "epoch: %4d" % trainer.totalepochs, \
" train error: %5.2f%%" % trnresult, \
" test error: %5.2f%%" % tstresult
predicted = np.array( trainer.testOnClassData(dataset=tstdata) )
real = np.array( tstdata['class'][:,0] )
# print real, predicted
# predicted = predicted[real==1]
# real = real[real==1]
print len(predicted), len(real)
print f1_score(predicted, real, average=None)
# if accuracy_score(real, predicted) > acc:
# acc = accuracy_score(real, predicted)
print f1_score(predicted, real, average=None)
# trainer.trainUntilConvergence()
# trnresult = percentError( trainer.testOnClassData(),
# trndata['class'] )
# tstresult = percentError( trainer.testOnClassData(
# dataset=tstdata ), tstdata['class'] )
# print "epoch: %4d" % trainer.totalepochs, \
# " train error: %5.2f%%" % trnresult, \
# " test error: %5.2f%%" % tstresult
print '---------------------------------'
# sys.exit()
pass
def execute_mlp(n_neurons, data_size, learn_rate, momentum_rate, f):
dic = {'Iris-setosa\n': 0, 'Iris-versicolor\n': 1, 'Iris-virginica\n': 2}
filename = "iris.txt"
file = read_file(filename)
file = change_class_name(file, dic)
file = str_to_number(file)
file_array = np.array(file)
data = normalize_data(file_array)
#data = order_data(data)
data = data[::-1] #INTERTENDO A ORDEM DOS ITENS
inputs = data[:, :-1] #COPIAR TODAS AS COLUNAS MENOS A ULTIMA
targets = data[:, -1] #COPIAR ULTIMA COLUNA
train_data_temp, test_data_temp = train_test_data(data, data_size)
train_data = ClassificationDataSet(
4, nb_classes=3) #TAMANHO DA ENTRADA, NUMERO DE CLASSES
test_data = ClassificationDataSet(
4, nb_classes=3) #TAMANHO DA ENTRADA, NUMERO DE CLASSES
cont = 0
for n in range(0, len(train_data_temp)):
train_data.addSample(train_data_temp[n][:-1], [train_data_temp[n][-1]])
#print(train_data.getSample(cont))
#cont = cont + 1
for n in range(0, len(test_data_temp)):
test_data.addSample(test_data_temp[n][:-1], [test_data_temp[n][-1]])
train_data._convertToOneOfMany()
test_data._convertToOneOfMany()
'''
print ("Number of training patterns: ", len(train_data))
print ("Input and output dimensions: ", train_data.indim, train_data.outdim)
print ("First sample (input, target, class):")
print (test_data['input'][0], test_data['target'][0], test_data['class'][0])
'''
network = FeedForwardNetwork()
inLayer = SigmoidLayer(train_data.indim)
first_hiddenLayer = SigmoidLayer(n_neurons)
second_hiddenLayer = SigmoidLayer(n_neurons)
outLayer = SigmoidLayer(train_data.outdim)
network.addInputModule(inLayer)
network.addModule(first_hiddenLayer)
network.addModule(second_hiddenLayer)
network.addOutputModule(outLayer)
in_to_hidden = FullConnection(inLayer, first_hiddenLayer)
hidden_to_hidden = FullConnection(first_hiddenLayer, second_hiddenLayer)
hidden_to_out = FullConnection(second_hiddenLayer, outLayer)
network.addConnection(in_to_hidden)
network.addConnection(hidden_to_hidden)
network.addConnection(hidden_to_out)
network.sortModules()
#trainer = BackpropTrainer( network, dataset=train_data, momentum=momentum_rate, verbose=False, weightdecay=learn_rate)
trainer = BackpropTrainer(network, dataset=train_data, verbose=False)
for i in range(1):
trainer.trainEpochs(1000)
result = trainer.testOnClassData(test_data, return_targets=True)
#result = classification(result[1],result[0])
print(result)
f.write(str(result))
f.flush()
fnn.sortModules()
# Train fnn using BP until convergence
trainer = BackpropTrainer(fnn, ds_train, learningrate = 0.01, verbose = True,
weightdecay = 0.1)
# batchlearning = True, weightdecay = 0.1, momentum
err_train, err_valid = trainer.trainUntilConvergence(maxEpochs = 1000)
# convergence curve
import matplotlib.pyplot as plt
plt.plot(err_train, 'b', err_valid, 'r')
plt.show()
# model testing
from pybrain.utilities import percentError
testResult = percentError(trainer.testOnClassData(), ds_test['target'])
print("epoch: %d" % trainer.totalepochs, "test error: %f%%" % testResult)
#%%
# Save model and result
NetworkWriter.writeToFile(fnn, '/Users/echozhao/Documents/WiDS/fnn.xml')
joblib.dump(sx, 'sx.pkl', compress = 3)
# Load Model
# fnn = NetworkReader.readFrom('/Users/echozhao/Documents/WiDS/fnn.xml')
# sx = joblib.load('sx.pkl')
#%%
# Doing Prediction
pred = np.array([fnn.activate(x) for x, _ in ds])
class testLearnedWeights(unittest.TestCase):
def setUp(self):
# self.net = buildNetwork(4, 6, 3, bias=False, inclass=TanhLayer,
# hiddenclass=TanhLayer, outclass=LinearLayer)
# self.net.sortModules()
self.net = createNN()
self.trn_d, self.tst_d = pybrainData(0.01)
self.trainer = BackpropTrainer(self.net,
dataset=self.trn_d,
learningrate=0.01,
momentum=0.1,
verbose=True,
weightdecay=0.0)
self.trainer.trainEpochs(1)
def testBPHasLearned(self):
trnresult = percentError(self.trainer.testOnClassData(),
self.trn_d['class'])
tstresult = percentError(
self.trainer.testOnClassData(dataset=self.tst_d),
self.tst_d['class'])
print 'trn perc error', trnresult
print 'tst perc error', tstresult
def testBPWeightsOnMyNetwork(self):
pyb_ws = self.net.params.copy()
pop = createPop()
for nn in pop:
nn.wi = pyb_ws[:nn.wi.size].reshape(NN.nh, NN.ni).T
nn.wo = pyb_ws[nn.wi.size:].reshape(NN.no, NN.nh).T
pairPop(pop, verbose=20)
def testWeightsAndActivationsEquivalent(self):
pyb_ws = self.net.params
nn = NN()
nn.wi = pyb_ws[:nn.wi.size].reshape(NN.nh, NN.ni).T
nn.wo = pyb_ws[nn.wi.size:].reshape(NN.no, NN.nh).T
for i, x in enumerate(self.trn_d['input']):
nn.activate(x)
out = self.net.activate(x)
npt.assert_array_equal(nn.ai, self.net['in'].outputbuffer[0])
# self.assertItemsEqual(list(nn.ah), list(self.net['hidden0'].outputbuffer[0]))
for j, pb_ah in enumerate(self.net['hidden0'].outputbuffer[0]):
self.assertAlmostEqual(nn.ah[j], pb_ah)
for k, pb_ao in enumerate(out):
self.assertAlmostEqual(nn.ao[k], pb_ao)
def testDataAssignedCorrectly(self):
NN.pat = zip(self.trn_d['input'], self.trn_d['target'])
pyb_ws = self.net.params.copy()
nn = NN()
nn.wi = pyb_ws[:nn.wi.size].reshape(NN.nh, NN.ni).T
nn.wo = pyb_ws[nn.wi.size:].reshape(NN.no, NN.nh).T
correct = 0
wrong = 0
all_aos = []
for i, x in enumerate(self.trn_d['input']):
nn.activate(x)
out = self.net.activate(x)
all_aos.append(nn.ao)
if not (out - self.trn_d['target'][i]).any():
correct += 1
else:
wrong += 1
for i in range(len(array(NN.pat)[:, 0])):
npt.assert_array_equal(self.trn_d['input'][i],
array(NN.pat)[:, 0][i])
npt.assert_array_equal(self.trn_d['input'][i],
array(nn.pat)[:, 0][i])
npt.assert_array_equal(self.trn_d['target'][i],
array(NN.pat)[:, 1][i])
npt.assert_array_equal(self.trn_d['target'][i],
array(nn.pat)[:, 1][i])
def testPercentErrorIsSame(self):
NN.pat = zip(self.trn_d['input'], self.trn_d['target'])
pyb_ws = self.net.params.copy()
nn = NN()
nn.wi = pyb_ws[:nn.wi.size].reshape(NN.nh, NN.ni).T
nn.wo = pyb_ws[nn.wi.size:].reshape(NN.no, NN.nh).T
correct = 0
wrong = 0
argmax_cor = 0
argmax_wng = 0
all_aos = []
for i, x in enumerate(self.trn_d['input']):
nn.activate(x)
out = self.net.activate(x)
# print 'ga bp trg', nn.ao, out, self.trn_d['target'][i], '++++' if not (out - self.trn_d['target'][i]).any() else '-'
all_aos.append(nn.ao.copy())
if not (out - self.trn_d['target'][i]).any():
correct += 1
else:
wrong += 1
if argmax(out) == argmax(self.trn_d['target'][i]):
argmax_cor += 1
else:
argmax_wng += 1
print 'actual', wrong, 'wrong', correct, 'correct', float(wrong) / (
wrong + correct) * 100
print 'using argmax', argmax_wng, 'wrong', argmax_cor, 'correct', float(
argmax_wng) / (argmax_wng + argmax_cor) * 100
argmax_perc_err = float(argmax_wng) / (argmax_wng + argmax_cor) * 100
res = nn.sumErrors()
nn_perc_err = 100 - res[1]
pb_nn_perc_err = percentError(self.trainer.testOnClassData(),
self.trn_d['class'])
self.assertAlmostEqual(nn_perc_err, pb_nn_perc_err)
self.assertAlmostEqual(nn_perc_err, pb_nn_perc_err, argmax_perc_err)
trainer = BackpropTrainer(fnn_backprop,
dataset=training_data,
momentum=0.1,
verbose=True,
weightdecay=0.01)
epochs = 10
trnerr_backprop = []
tsterr_backprop = []
for i in xrange(epochs):
# If you set the 'verbose' trainer flag, this will print the total error as it goes.
trainer.trainEpochs(1)
#output_train = testOnClassData_custom(fnn_backprop, dataset=training_data)
output_train = trainer.testOnClassData()
for tr in output_train:
#print("This is the training output value: ", tr)
continue
trnresult = percentError(trainer.testOnClassData(), training_data['class'])
tstresult = percentError(trainer.testOnClassData(dataset=test_data),
test_data['class'])
print("epoch: %4d" % trainer.totalepochs,
" train error: %5.2f%%" % trnresult,
" test error: %5.2f%%" % tstresult)
trnerr_backprop.append(trnresult)
tsterr_backprop.append(tstresult)
#*****************End of Backpropagation NN*******************************
fnn_rhc = buildNetwork(training_data.indim,
from sklearn import datasets
olivetti = datasets.fetch_olivetti_faces()
X, y = olivetti.data, olivetti.target
from pybrain.datasets import ClassificationDataSet
from pybrain.utilities import percentError
from pybrain.tools.shortcuts import buildNetwork
from pybrain.supervised.trainers import BackpropTrainer
from pybrain.structure.modules import SoftmaxLayer
ds = ClassificationDataSet(4096, 1, nb_classes=40)
for k in xrange(len(X)):
ds.addSample(X[k], y[k])
tstdata, trndata = ds.splitWithProportion(0.25)
trndata._convertToOneOfMany()
tstdata._convertToOneOfMany()
fnn = buildNetwork(trndata.indim, 64, trndata.outdim, outclass=SoftmaxLayer)
trainer = BackpropTrainer(fnn,
dataset=trndata,
momentum=0.1,
learningrate=0.01,
verbose=True,
weightdecay=0.01)
trainer.trainEpochs(50)
print 'Percent Error on Test dataset: ', percentError(
trainer.testOnClassData(dataset=tstdata), tstdata['class'])
print "Loading Network File"
networkFile = open('trainedNet1.cpkl', 'r')
network = cPickle.load(networkFile)
networkFile.close()
# Load in the data
dataFile = open(path.join('cifar-10-batches-py', 'test_batch'), 'r')
data = cPickle.load(dataFile)
images = np.array(data['data'])
labels = np.array(data['labels'])
dataFile.close()
# Construct the classification data set for evaluation
print 'Constructing the Data Set'
dataSet = ClassificationDataSet(3072, 1, nb_classes=10)
for index in range(0, labels.size):
dataSet.addSample(images[index], labels[index])
dataSet._convertToOneOfMany()
# Create the trainer to use to evaluate the existing network
print 'Creating the Trainer'
trainer = BackpropTrainer(network)
# Evaluate the network against the test data
print 'Testing the data'
error = percentError(trainer.testOnClassData(dataset=dataSet),
dataSet['class'])
print 'Data had', error, 'error'
def neural_network(data_model, classes, runs):
# Python brain
from pybrain.structure import FullConnection, FeedForwardNetwork, LinearLayer, SigmoidLayer, SoftmaxLayer
from pybrain.datasets import ClassificationDataSet
from pybrain.utilities import percentError
from pybrain.supervised.trainers import BackpropTrainer
from pybrain.tools.xml.networkwriter import NetworkWriter
from pybrain.tools.xml.networkreader import NetworkReader
import csv
# Build Network
try:
n = NetworkReader.readFrom('resources/net.xml')
print 'Loading previous network'
except:
print 'Generating new network'
# Create a new Network
n = FeedForwardNetwork()
# Define the input layer
inLayer = LinearLayer(len(data_model[0][0]))
# Define a hidden layer
hiddenLayer = SigmoidLayer(10)
hiddenLayer2 = SigmoidLayer(10)
# Define the output layer
outLayer = LinearLayer(classes)
# Add layers to network n
n.addInputModule(inLayer)
n.addModule(hiddenLayer)
n.addModule(hiddenLayer2)
n.addOutputModule(outLayer)
# Create layers
in_to_hidden = FullConnection(inLayer, hiddenLayer)
hidden_to_hidden2 = FullConnection(hiddenLayer, hiddenLayer2)
hidden2_to_out = FullConnection(hiddenLayer2, outLayer)
# Add connectors to network n
n.addConnection(in_to_hidden)
n.addConnection(hidden_to_hidden2)
n.addConnection(hidden2_to_out)
# Finish Network
n.sortModules()
# Other Stuff
ds = ClassificationDataSet(len(data_model[0][0]), 1, nb_classes=classes)
# di = ClassificationDataSet(2,1,0)
for o in data_model:
ds.addSample(o[0], o[1])
testing_data, training_data = ds.splitWithProportion(0.3)
training_data._convertToOneOfMany()
testing_data._convertToOneOfMany()
print "Number of training patterns: ", len(training_data)
print "Input and output dimensions: ", training_data.indim, training_data.outdim
print "First sample (input, target, class):"
print training_data['input'][0], training_data['target'][0], training_data[
'class'][0]
trainer = BackpropTrainer(n, dataset=training_data)
smart = []
dumb = []
with open("resources/minimum_error.csv", 'rb') as f:
reader = csv.reader(f)
for row in reader:
smart.append(row)
smart[0] = float(smart[0][0])
print 'The minimum error from previous runs =', smart[0]
for t in range(runs):
trainer.trainEpochs(1)
trnresult = percentError(trainer.testOnClassData(),
training_data['class'])
tstresult = percentError(trainer.testOnClassData(dataset=testing_data),
testing_data['class'])
print "epoch: %4d" % trainer.totalepochs, " train error: %5.5f%%" % trnresult, " test error: %5.5f%%" % tstresult
smart.append(tstresult)
if tstresult <= min(smart):
NetworkWriter.writeToFile(n, 'resources/net.xml')
print 'Best!'
else:
dumb.append('1')
print 'Worst!'
minimum_error = []
minimum_error.append(min(smart))
with open("resources/minimum_error.csv", 'wb') as f:
writer = csv.writer(f)
writer.writerow(minimum_error)
print 'Minimum error (current state)', min(smart)
return n
PyBDataTrain_nn.outdim,
bias=True,
outclass=SoftmaxLayer)
trainer = BackpropTrainer(fnn,
dataset=PyBDataTrain_nn,
momentum=0.1,
verbose=True,
weightdecay=0.01)
epochs = 6
trnerr = []
tsterr = []
for i in xrange(epochs):
# If you set the 'verbose' trainer flag, this will print the total error as it goes.
trainer.trainEpochs(3)
trnresult = percentError(trainer.testOnClassData(),
PyBDataTrain_nn['class'])
tstresult = percentError(trainer.testOnClassData(dataset=PyBDataTest_nn),
PyBDataTest_nn['class'])
print("epoch: %4d" % trainer.totalepochs,
" train error: %5.2f%%" % trnresult,
" test error: %5.2f%%" % tstresult)
trnerr.append(trnresult)
tsterr.append(tstresult)
fig_nn = plt.figure()
ax = fig_nn.add_subplot(1, 1, 1)
ax.set_title("Neural Network Convergence")
ax.set_xlabel('Epoch')
ax.set_ylabel('Error')
ax.semilogy(range(len(trnerr)), trnerr, 'b', range(len(tsterr)), tsterr, 'r')
# trainer = BackpropTrainer(net, trndata, batchlearning=True)
# trainer.trainEpochs(50)
# err_train, err_valid = trainer.trainUntilConvergence(maxEpochs=50)
# convergence curve for accumulative BP algorithm process
# import matplotlib.pyplot as plt
# plt.plot(err_train,'b',err_valid,'r')
# plt.title('BP network classification')
# plt.ylabel('accuracy')
# plt.xlabel('epochs')
# plt.show()
# 1.3 model testing
from pybrain.utilities import percentError
tstresult = percentError(trainer.testOnClassData(tstdata), tstdata['class'])
print("epoch: %4d" % trainer.totalepochs, " test error: %5.2f%%" % tstresult)
er_sum = 0
for i in range(20):
# generation of train set and test set (3:1)
tstdata_temp, trndata_temp = ds.splitWithProportion(0.25)
tstdata = ClassificationDataSet(19, 1, nb_classes=2, class_labels=label)
for n in range(0, tstdata_temp.getLength()):
tstdata.appendLinked(
tstdata_temp.getSample(n)[0],
tstdata_temp.getSample(n)[1])
trndata = ClassificationDataSet(19, 1, nb_classes=2, class_labels=label)
for n in range(0, trndata_temp.getLength()):
trndata.appendLinked(
trndata_temp.getSample(n)[0],
dataSet = ClassificationDataSet(4, 1, nb_classes=3)
for i in range(len(dataFeatures)):
dataSet.addSample(np.ravel(dataFeatures[i]), dataTargets[i])
trainingData, testData = splitWithProportion(dataSet, 0.7)
trainingData._convertToOneOfMany()
testData._convertToOneOfMany()
neuralNetwork = buildNetwork(trainingData.indim,
7,
trainingData.outdim,
outclass=SoftmaxLayer)
trainer = BackpropTrainer(neuralNetwork,
dataset=trainingData,
momentum=0.01,
learningrate=0.05,
verbose=True)
trainer.trainEpochs(10000)
print(
'Error (test dataset): ',
percentError(trainer.testOnClassData(dataset=testData), testData['class']))
print('\n\n')
counter = 0
for input in dataFeatures:
print(counter, " output is according to the NN: ",
neuralNetwork.activate(input))
counter = counter + 1
err_train, err_valid = trainer.trainUntilConvergence(maxEpochs = 1000,
validationProportion = 0.25)
#If no dataset is given, the dataset passed during Trainer initialization is
#used. validationProportion is the ratio of the dataset that is used for the
#validation dataset.
#Convergence curve for accumulative BP algorithm process
f3 = plt.figure(3)
plt.plot(err_train, 'b', err_valid, 'r')
plt.title('BP network classification')
plt.ylabel('error rate')
plt.xlabel('epochs')
plt.show()
#testing
tst_result = percentError(trainer.testOnClassData(tstdata),
tstdata['class'])
#Return percentage of mismatch between out and target values
print("epoch: %4d" % trainer.totalepochs, " test error: %5.2f%%" % tst_result)
#%%Decision tree
import pandas as pd
X_train, X_test, y_train, y_test = train_test_split(X, y,
test_size = 0.5,
random_state = 0)
#random_state: int or RandomState
#Pseudo-random number generator state used for random sampling
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train,
test_size = 0.5,
def perceptron(hidden_neurons=5, weightdecay=0.01, momentum=0.1):
INPUT_FEATURES = 2
CLASSES = 3
HIDDEN_NEURONS = hidden_neurons
WEIGHTDECAY = weightdecay
MOMENTUM = momentum
# Generate the labeled set
g = generate_data()
#g = generate_data2()
alldata = g['d']
minX, maxX, minY, maxY = g['minX'], g['maxX'], g['minY'], g['maxY']
# Split data into test and training dataset
tstdata, trndata = alldata.splitWithProportion(0.25)
trndata._convertToOneOfMany() # This is necessary, but I don't know why
tstdata._convertToOneOfMany() # http://stackoverflow.com/q/8154674/562769
print("Number of training patterns: %i" % len(trndata))
print("Input and output dimensions: %i, %i" %
(trndata.indim, trndata.outdim))
print("Hidden neurons: %i" % HIDDEN_NEURONS)
print("First sample (input, target, class):")
print(trndata['input'][0], trndata['target'][0], trndata['class'][0])
fnn = buildNetwork(trndata.indim,
HIDDEN_NEURONS,
trndata.outdim,
outclass=SoftmaxLayer)
trainer = BackpropTrainer(fnn,
dataset=trndata,
momentum=MOMENTUM,
verbose=True,
weightdecay=WEIGHTDECAY)
# Visualization
ticksX = arange(minX - 1, maxX + 1, 0.2)
ticksY = arange(minY - 1, maxY + 1, 0.2)
X, Y = meshgrid(ticksX, ticksY)
# need column vectors in dataset, not arrays
griddata = ClassificationDataSet(INPUT_FEATURES, 1, nb_classes=CLASSES)
for i in range(X.size):
griddata.addSample([X.ravel()[i], Y.ravel()[i]], [0])
for i in range(20):
trainer.trainEpochs(1)
trnresult = percentError(trainer.testOnClassData(), trndata['class'])
tstresult = percentError(trainer.testOnClassData(dataset=tstdata),
tstdata['class'])
print("epoch: %4d" % trainer.totalepochs,
" train error: %5.2f%%" % trnresult,
" test error: %5.2f%%" % tstresult)
out = fnn.activateOnDataset(griddata)
# the highest output activation gives the class
out = out.argmax(axis=1)
out = out.reshape(X.shape)
figure(1) # always print on the same canvas
ioff() # interactive graphics off
clf() # clear the plot
for c in [0, 1, 2]:
here, _ = where(tstdata['class'] == c)
plot(tstdata['input'][here, 0], tstdata['input'][here, 1], 'o')
if out.max() != out.min(): # safety check against flat field
contourf(X, Y, out) # plot the contour
ion() # interactive graphics on
draw() # update the plot
ioff()
show()
# trndata.indim, 6, trndata.outdim, recurrent=False, bias=False
# )
trainer = BackpropTrainer(
net, dataset=trndata, learningrate=0.01, momentum=0.5, verbose=True
)
print('')
for i in range(50):
# train the network for 1 epoch
trainer.trainEpochs(1)
# evaluate the result on the training and test data
trnresult = percentError(
trainer.testOnClassData(dataset=trndata),
trndata['class']
)
# print the result
print(
'epoch: %4d' % trainer.totalepochs,
' train error: %5.2f%%' % trnresult
)
# single sample test
test_input = [0, 231, 190]
# activate the network (feedforward) using this one sample
test_output = net.activate(test_input)
# print output
print(
def exec_algo(xml_file, output_location):
rootObj = ml.parse(xml_file)
#Getting the root element so that we get the subclasses and its members and member function
xmlParamDetails = rootObj.MachineLearning.classification
#Gather param values from the XML parsed object
file = open(xmlParamDetails.datafile)
var_inp = xmlParamDetails.input
var_out = xmlParamDetails.output
classes = xmlParamDetails.classes
split = xmlParamDetails.split
learningrate = xmlParamDetails.algorithm.MultiLayerPerceptron.learningRate
momentum = xmlParamDetails.algorithm.MultiLayerPerceptron.momentum
epochs = xmlParamDetails.algorithm.MultiLayerPerceptron.epochs
hiddenNeurons = int(
xmlParamDetails.algorithm.MultiLayerPerceptron.hiddenLayers)
hiddenLayer = xmlParamDetails.algorithm.MultiLayerPerceptron.hiddenLayerActivation
outputLayer = xmlParamDetails.algorithm.MultiLayerPerceptron.outputLayerActivation
delimiter = xmlParamDetails.delimiter
DS = ClassificationDataSet(var_inp, var_out, nb_classes=classes)
for line in file.readlines():
data = [float(x) for x in line.strip().split(',') if x != '']
inp = tuple(data[:var_inp])
output = tuple(data[var_inp:])
DS.addSample(inp, output)
tstdata, trndata = DS.splitWithProportion(split)
trdata = ClassificationDataSet(trndata.indim, var_out, nb_classes=classes)
tsdata = ClassificationDataSet(tstdata.indim, var_out, nb_classes=classes)
for i in xrange(trndata.getLength()):
trdata.addSample(trndata.getSample(i)[0], trndata.getSample(i)[1])
for i in xrange(tstdata.getLength()):
tsdata.addSample(tstdata.getSample(i)[0], tstdata.getSample(i)[1])
trdata._convertToOneOfMany()
tsdata._convertToOneOfMany()
fnn = FeedForwardNetwork()
inputLayer = LinearLayer(trdata.indim)
if hiddenLayer == 'Sigmoid':
hiddenLayer = SigmoidLayer(hiddenNeurons)
elif hiddenLayer == 'Softmax':
hiddenLayer = SoftmaxLayer(hiddenNeurons)
else:
hiddenLayer = LinearLayer(hiddenNeurons)
if outputLayer == 'Sigmoid':
outputLayer = SigmoidLayer(trdata.outdim)
elif outputLayer == 'Softmax':
outputLayer = SoftmaxLayer(trdata.outdim)
else:
outputLayer = LinearLayer(trdata.outdim)
fnn.addInputModule(inputLayer)
fnn.addModule(hiddenLayer)
fnn.addOutputModule(outputLayer)
in_to_hidden = FullConnection(inputLayer, hiddenLayer)
hidden_to_outputLayer = FullConnection(hiddenLayer, outputLayer)
fnn.addConnection(in_to_hidden)
fnn.addConnection(hidden_to_outputLayer)
fnn.sortModules()
trainer = BackpropTrainer(fnn,
dataset=trdata,
verbose=True,
learningrate=learningrate,
momentum=momentum)
trainer.trainEpochs(epochs=epochs)
trresult = percentError(trainer.testOnClassData(), trdata['class'])
print("Training accuracy : %f " % (100 - trresult))
ts = time.time()
directory = output_location + sep + str(int(ts))
makedirs(directory)
fileObject = open(
output_location + sep + str(int(ts)) + sep + 'pybrain_MLP', 'w')
pickle.dump(trainer, fileObject)
pickle.dump(fnn, fileObject)
fileObject.close()
# ds = input and output desired
# We multiply the weights by "weightdecay" to keep themf rom growing too large
# -- This is a regularization method
trainer = BackpropTrainer(net, ds, momentum=0.1, weightdecay=0.01)
# # Train the network once
# errorVal = trainer.train()
# print(errorVal)
# Train the network
numEpochs = 10
import sys
for i in range(numEpochs):
errorVals = trainer.train()
# Print how network is doing so far
trnresult = percentError(trainer.testOnClassData(), ds['class'])
# print("Epochs:", trainer.totalepochs)
#print("Percent error on training data:", trnresult)
sys.stdout.write(str(trnresult) + "%\n") # same as print
sys.stdout.flush()
# Plot results (based on example_fnn.py from PyBrain examples)
griddata, X, Y = generateGridData([-2., 2.5, 0.2], [-2., 2.5, 0.2])
out = net.activateOnDataset(griddata)
out = out.argmax(axis=1)
out = out.reshape(X.shape)
figure(1)
ioff() # interactive graphics off
clf()
plotData(ds)
def main():
trndata, tstdata = createDS()
for repeat in xrange(repeats):
print 'trial', repeat
iter_trn_results = []
iter_tst_results = []
nn = createNN(4, 6, 3)
nn.randomize()
hiddenAstrocyteLayer, outputAstrocyteLayer = \
associateAstrocyteLayers(nn)
trainer = BackpropTrainer(nn,
dataset=trndata,
learningrate=0.01,
momentum=0.1,
verbose=False,
weightdecay=0.0)
for grand_iter in xrange(iterations):
if grand_iter == 0:
trainer.train()
# trainNGA(nn, trndata, hiddenAstrocyteLayer, outputAstrocyteLayer)
trainer.train()
trnresult = percentError(trainer.testOnClassData(),
trndata['class'])
iter_trn_results.append(trnresult)
tstresult = percentError(trainer.testOnClassData(dataset=tstdata),
tstdata['class'])
iter_tst_results.append(tstresult)
if not grand_iter % 100:
print 'epoch %4d' % trainer.totalepochs, 'train error %5.2f%%'\
% trnresult, 'test error %5.2f%%' % tstresult
# MAKE SURE NOT IN ITER LOOP
all_trn_results.append(iter_trn_results)
all_tst_results.append(iter_tst_results)
assert array(iter_trn_results).shape == (iterations, ), \
array(iter_trn_results).shape
assert array(iter_tst_results).shape == (iterations, ), \
array(iter_tst_results).shape
assert array(all_trn_results).shape == (repeats, iterations), \
array(all_trn_results).shape
assert array(all_tst_results).shape == (repeats, iterations), \
array(all_tst_results).shape
a = datetime.datetime.now().utctimetuple()
time_string = str(a[3]) + str(a[4]) + '_' + str(a[2]) + '-' + \
str(a[1]) + '-' + str(a[0])
if os.environ['OS'] == 'Windows_NT':
sep = '\\'
else:
sep = '/'
pybrain_dir = os.getcwd() + sep
assert pybrain_dir[-10:-1] == 'mypybrain', \
'is actually this ' + pybrain_dir[-10:-1]
os.mkdir(pybrain_dir + 'experiment_results' + sep + time_string)
trnf = open(
pybrain_dir + 'experiment_results' + sep + time_string +
'/all_trn_results.out', 'w')
np.savetxt(trnf, all_trn_results)
tstf = open(
pybrain_dir + 'experiment_results' + sep + time_string +
'/all_tst_results.out', 'w')
np.savetxt(tstf, all_tst_results)
test_data
# pybrain's version of dummy variables
test_data._convertToOneOfMany()
training_data._convertToOneOfMany()
training_data['input']
training_data['target']
test_data.indim
test_data.outdim
# instantiate the model
fnn = buildNetwork(training_data.indim,
64,
training_data.outdim,
outclass=SoftmaxLayer)
trainer = BackpropTrainer(fnn,
dataset=training_data,
momentum=0.1,
learningrate=0.01,
verbose=True,
weightdecay=0.01)
# change the number of eopchs to try to get better results!
trainer.trainEpochs(10)
print 'Percent Error on Test dataset: ' , \
percentError( trainer.testOnClassData (
dataset=test_data )
, test_data['class'] )
