def fit(self, ds, epochs=100, hiddenSize=100, initialLearningrate=0.002, decay=0.9999, myWeightdecay=0.8, plot=False, testDs=None, momentum=0): firstSample = ds.getSample(0) print firstSample inputSize, hiddenSize, outputSize = len(firstSample[0]), hiddenSize, len(firstSample[1]) inLayer = LinearLayer(inputSize) hiddenLayer =SigmoidLayer(hiddenSize) outLayer = LinearLayer(outputSize) n = FeedForwardNetwork() n.addInputModule(inLayer) n.addModule(hiddenLayer) b = BiasUnit() n.addModule(b) n.addOutputModule(outLayer) in_to_hidden = FullConnection(inLayer, hiddenLayer) hidden_to_out = FullConnection(hiddenLayer, outLayer) b_to_hidden = FullConnection(b, hiddenLayer) b_to_out = FullConnection(b, outLayer) n.addConnection(in_to_hidden) n.addConnection(hidden_to_out) n.addConnection(b_to_hidden) n.addConnection(b_to_out) n.sortModules() self.supervisedNet = n n = self.supervisedNet self.supervisedTrainer = BackpropTrainer(n, ds, learningrate=initialLearningrate, lrdecay=decay, verbose=True, weightdecay=myWeightdecay, batchlearning=True, momentum=momentum) """ #supervisedTrainer.trainEpochs(epochs) def eval(representationNet, output, target): output = [1 if o>0.5 else 0 for o in output] output = np.array(output) target = np.array(target) assert len(output) == len(target) n_correct = sum( output == target ) return float(n_correct) / float(len(output)) """ #cv = CrossValidator(self.supervisedTrainer, ds,n_folds=int(len(ds)/5)) #valfunc=eval) Y = np.array([y for x,y in ds]) if plot: cvResults = [] trainResults = [] totalF1s=[] totalTestf1 = [] sums=[] lastTrainVec = np.zeros(Y.shape) for epochNum in range(epochs): self.supervisedTrainer.train() """ pred = n.activateOnDataset(ds) f1s = [] for col in range(pred.shape[1]): _, bestF1 = labanUtil.getSplitThreshold(pred[:, col], Y[:, col]) f1s.append(bestF1) cvResults.append(np.mean(f1s)) """ #cvResults.append(cv.validate()) trainVec = n.activateOnDataset(ds) #print trainVec[0] trainDif = np.abs(np.subtract(Y, trainVec)) difdif = np.abs(np.subtract(lastTrainVec, trainVec)) lastTrainVec = copy.deepcopy(trainVec) #print trainDif trainRes = float(sum(sum(trainDif)))/Y.shape[0]/Y.shape[1] print 'epoch num:', epochNum print 'trainDif sum: ', sum(sum(trainDif)) print 'trainVec sum: ', sum(sum(np.abs(trainVec))) print 'difdif sum: ', sum(sum(difdif)) print 'hiddenSize: ', hiddenSize print 'initialLearningrate', initialLearningrate print 'decay', decay print 'myWeightdecay', myWeightdecay print 'momentum', momentum s = sum(np.abs(trainVec[0])) s2 = sum(Y[0]) print 'sum(trainVec[0])', s print 'sum(Y[0])', sum(Y[0]) trainResults.append(trainRes) sums.append(s2) splits = [] for col in range(trainVec.shape[1]): bestSplit, bestF1 = labanUtil.getSplitThreshold(trainVec[:, col], Y[:, col]) splits.append(bestSplit) if not testDs is None: testPred = np.array(n.activateOnDataset(testDs)) Y_test = np.array([y for x,y in testDs]) for col in range(trainVec.shape[1]): if not testDs is None: testPred[:, col] = [1 if e>=splits[col] else 0 for e in testPred[:, col]] trainVec[:, col] = [1 if e>=splits[col] else 0 for e in trainVec[:, col]] f1s = [] testf1s = [] for j in range(Y.shape[1]): f1s.append(metrics.f1_score(Y[:,j], trainVec[:,j])) if not testDs is None: testf1s.append(metrics.f1_score(Y_test[:,j], testPred[:,j])) totalF1s.append(np.mean(f1s)) if not testDs is None: totalTestf1.append(np.mean(testf1s)) #plt.title() des= 'Hidden size: ' + str(hiddenSize)\ +', epochs: '+ str(epochs) \ +', initialLearningrate: '+str(initialLearningrate) \ +', decay: '+str(decay) \ +', momentum: '+str(momentum) \ +'\n myWeightdecay: '+str(myWeightdecay) \ +', hidden function: '+hiddenLayer.name \ +', output function: '+outLayer.name \ +'\n trainDsSize: '+str(len(ds)) \ +', testDsSize: '+str(len(testDs)) \ +', best train f1: '+str(max(totalF1s)) \ +', accuracy: '+str(min(trainResults)) if not testDs is None: des+= ', best test f1: '+str(max(totalTestf1)) plt.plot(range(epochs), totalF1s, label='Train F1') plt.plot(range(epochs), trainResults, label='accuracy') if not testDs is None: plt.plot(range(epochs), totalTestf1, label='Test F1') plt.legend() an = FeedForwardNetwork() an.addInputModule(inLayer) an.addOutputModule(hiddenLayer) an.addModule(b) an.addConnection(in_to_hidden) an.addConnection(b_to_hidden) an.sortModules() self.representationNet = an return des
def brescia_nn(train, test, max_epochs=None, verbose=False): trainval_ds = SupervisedDataSet(5, 1) test_ds = SupervisedDataSet(5, 1) for datum in train: trainval_ds.addSample(datum[:5], (datum[5],)) for datum in test: test_ds.addSample(datum[:5], (datum[5],)) train_ds, val_ds = trainval_ds.splitWithProportion(0.75) if verbose: print "Train, validation, test:", len(train_ds), len(val_ds), len(test_ds) ns = {} min_error = -1 min_h = -1 # use validation to form 4-layer network with two hidden layers, # with (2n + 1) nodes in the first hidden layer and somewhere from # 1 to (n - 1) in the second hidden layer for h2 in range(1, 5): if verbose: start = time.time() print "h2 nodes:", h2 # create the network if verbose: print "building network" n = FeedForwardNetwork() inLayer = LinearLayer(5) hiddenLayer1 = SigmoidLayer(11) hiddenLayer2 = SigmoidLayer(h2) outLayer = LinearLayer(1) n.addInputModule(inLayer) n.addModule(hiddenLayer1) n.addModule(hiddenLayer2) n.addOutputModule(outLayer) in_to_hidden = FullConnection(inLayer, hiddenLayer1) hidden_to_hidden = FullConnection(hiddenLayer1, hiddenLayer2) hidden_to_out = FullConnection(hiddenLayer2, outLayer) n.addConnection(in_to_hidden) n.addConnection(hidden_to_hidden) n.addConnection(hidden_to_out) n.sortModules() # training if verbose: print "beginning training" trainer = BackpropTrainer(n, train_ds) trainer.trainUntilConvergence(maxEpochs=max_epochs) ns[h2] = n # validation if verbose: print "beginning validation" out = n.activateOnDataset(val_ds) actual = val_ds['target'] error = np.sqrt(np.sum((out - actual)**2) / len(val_ds)) if verbose: print "RMSE:", error if min_error == -1 or error < min_error: min_error = error min_h = h2 if verbose: stop = time.time() print "Time:", stop - start # iterate through if verbose: print "best number of h2 nodes:", min_h out_test = ns[min_h].activateOnDataset(test_ds) return ns[h2], out_test
feed_forward_net.addConnection(bias_to_hidden) feed_forward_net.addConnection(bias_to_out) """ feed_forward_net.sortModules() # fnn = buildNetwork( trndata.indim, 20, trndata.outdim, bias = True, outclass=SigmoidLayer, hiddenclass=TanhLayer) trainer = BackpropTrainer(feed_forward_net, dataset=trndata, momentum=0.1, verbose=True, weightdecay=0.01) trainer.trainUntilConvergence(validationProportion=0.25, maxEpochs=100, continueEpochs=5) trnresult = percentError(trainer.testOnClassData(), trndata["class"]) tstresult = percentError(trainer.testOnClassData(dataset=tstdata), tstdata["class"]) print "epoch: %4d" % trainer.totalepochs, " train error: %5.2f%%" % trnresult, " test error: %5.2f%%" % tstresult out = feed_forward_net.activateOnDataset(tstdata) print out.shape sur_test = [] for row in out: if row[0] >= 0.5: row[0] = 1 row[1] = 0 else: row[0] = 0 row[1] = 1 # print out # print sur_test # print sum(sur_test)
# #### Build and use the Trainer # In[ ]: #Learning Rate shouldnt be bigger than 0.01 t = BackpropTrainer(fnn, learningrate = 0.01, momentum = 0.99, verbose = True,lrdecay=0.9999) #Training on the DataSet with 1500 epochs t.trainOnDataset(DS, 1500) # # Model Evaluation # In[ ]: y_pred = fnn.activateOnDataset(DS) #"DeNormalize" Again to turn the data into the original monetary value y_pred = y_pred * wy # In[ ]: #Create the DataSet for a RegressionLine x_pred = np.arange(5,870, 10) DS_Eval = SupervisedDataSet( 1, 0 ) #Append Linked: x,y for i in range(len(x_pred)):
class NeuralNetworkClassification(algorithmbase): def ExtraParams(self, hiddenlayerscount, hiddenlayernodescount): self.hiddenlayerscount = hiddenlayerscount self.hiddenlayernodescount = hiddenlayernodescount return self def PreProcessTrainData(self): self.traindata = preprocess_apply(self.traindata, self.missingvaluemethod, self.preprocessingmethods) def PrepareModel(self, savedmodel = None): if savedmodel != None: self.trainer = savedmodel else: attributescount=len(self.traindata[0]) nrclass = len(set(self.trainlabel)) self.ds = ClassificationDataSet(attributescount, target=nrclass, nb_classes=nrclass, class_labels=list(set(self.trainlabel))) for i in range(len(self.traindata)): self.ds.appendLinked(self.traindata[i], [self.trainlabel[i]]) self.ds._convertToOneOfMany() self.net = FeedForwardNetwork() inLayer = LinearLayer(len(self.traindata[0])) self.net.addInputModule(inLayer) hiddenLayers=[] for i in range(self.hiddenlayerscount): hiddenLayer=SigmoidLayer(self.hiddenlayernodescount) hiddenLayers.append(hiddenLayer) self.net.addModule(hiddenLayer) outLayer = SoftmaxLayer(nrclass) self.net.addOutputModule(outLayer) layers_connections=[] layers_connections.append(FullConnection(inLayer, hiddenLayers[0])) for i in range(self.hiddenlayerscount-1): layers_connections.append(FullConnection(hiddenLayers[i-1], hiddenLayers[i])) layers_connections.append(FullConnection(hiddenLayers[-1], outLayer)) for layers_connection in layers_connections: self.net.addConnection(layers_connection) self.net.sortModules() #training the network self.trainer = BackpropTrainer(self.net, self.ds) self.trainer.train() def PreProcessTestDate(self): self.testdata=preprocess_apply(self.testdata, self.missingvaluemethod, self.preprocessingmethods) def Predict(self): prediction=[] attributescount=len(self.testdata[0]) nrclass = len(set(self.testlabel)) dstraindata = ClassificationDataSet(attributescount, target=nrclass, nb_classes=nrclass, class_labels=list(set(self.testlabel))) for i in range(len(self.testdata)): dstraindata.appendLinked(self.testdata[i], self.testlabel[i]) dstraindata._convertToOneOfMany() out = self.net.activateOnDataset(dstraindata) prediction = out.argmax(axis=1) ''' for testrecord in self.testdata : out = self.net.activate(testrecord)[0] prediction.append(out) ''' self.result = [self.testlabel, prediction] def GetModel(self): return self.trainer
if os.path.isfile(filetoopen): myfile = open('myparam2.txt','r') c=[] for line in myfile: c.append(float(line)) n._setParameters(c) else: myfile = open('myparam2.txt','w') for i in n.params: myfile.write(str(i)+'\n') myfile.close() #activate the neural networks act = SupervisedDataSet(1,1) act.addSample((0.2,),(0.880422606518061,)) n.activateOnDataset(act) #create the test DataSet x = numpy.arange(0.0, 1.0+0.01, 0.01) s = 0.5+0.4*numpy.sin(2*numpy.pi*x) tsts = SupervisedDataSet(1,1) tsts.setField('input',x.reshape(len(x),1)) tsts.setField('target',s.reshape(len(s),1)) #read the train DataSet from file trndata = SupervisedDataSet.loadFromFile(os.path.join(os.getcwd(),'trndata')) #create the trainer t = BackpropTrainer(n, learningrate = 0.01 , momentum = mom) #train the neural network from the train DataSet
def brescia_nn(train, test, max_epochs=None, verbose=False): trainval_ds = SupervisedDataSet(5, 1) test_ds = SupervisedDataSet(5, 1) for datum in train: trainval_ds.addSample(datum[:5], (datum[5], )) for datum in test: test_ds.addSample(datum[:5], (datum[5], )) train_ds, val_ds = trainval_ds.splitWithProportion(0.75) if verbose: print "Train, validation, test:", len(train_ds), len(val_ds), len( test_ds) ns = {} min_error = -1 min_h = -1 # use validation to form 4-layer network with two hidden layers, # with (2n + 1) nodes in the first hidden layer and somewhere from # 1 to (n - 1) in the second hidden layer for h2 in range(1, 5): if verbose: start = time.time() print "h2 nodes:", h2 # create the network if verbose: print "building network" n = FeedForwardNetwork() inLayer = LinearLayer(5) hiddenLayer1 = SigmoidLayer(11) hiddenLayer2 = SigmoidLayer(h2) outLayer = LinearLayer(1) n.addInputModule(inLayer) n.addModule(hiddenLayer1) n.addModule(hiddenLayer2) n.addOutputModule(outLayer) in_to_hidden = FullConnection(inLayer, hiddenLayer1) hidden_to_hidden = FullConnection(hiddenLayer1, hiddenLayer2) hidden_to_out = FullConnection(hiddenLayer2, outLayer) n.addConnection(in_to_hidden) n.addConnection(hidden_to_hidden) n.addConnection(hidden_to_out) n.sortModules() # training if verbose: print "beginning training" trainer = BackpropTrainer(n, train_ds) trainer.trainUntilConvergence(maxEpochs=max_epochs) ns[h2] = n # validation if verbose: print "beginning validation" out = n.activateOnDataset(val_ds) actual = val_ds['target'] error = np.sqrt(np.sum((out - actual)**2) / len(val_ds)) if verbose: print "RMSE:", error if min_error == -1 or error < min_error: min_error = error min_h = h2 if verbose: stop = time.time() print "Time:", stop - start # iterate through if verbose: print "best number of h2 nodes:", min_h out_test = ns[min_h].activateOnDataset(test_ds) return ns[h2], out_test
class NeuralNetworkClassification(algorithmbase): def ExtraParams(self, hiddenlayerscount, hiddenlayernodescount): self.hiddenlayerscount = hiddenlayerscount self.hiddenlayernodescount = hiddenlayernodescount return self def PreProcessTrainData(self): self.traindata = preprocess_apply(self.traindata, self.missingvaluemethod, self.preprocessingmethods) def PrepareModel(self, savedmodel=None): if savedmodel != None: self.trainer = savedmodel else: attributescount = len(self.traindata[0]) nrclass = len(set(self.trainlabel)) self.ds = ClassificationDataSet(attributescount, target=nrclass, nb_classes=nrclass, class_labels=list( set(self.trainlabel))) for i in range(len(self.traindata)): self.ds.appendLinked(self.traindata[i], [self.trainlabel[i]]) self.ds._convertToOneOfMany() self.net = FeedForwardNetwork() inLayer = LinearLayer(len(self.traindata[0])) self.net.addInputModule(inLayer) hiddenLayers = [] for i in range(self.hiddenlayerscount): hiddenLayer = SigmoidLayer(self.hiddenlayernodescount) hiddenLayers.append(hiddenLayer) self.net.addModule(hiddenLayer) outLayer = SoftmaxLayer(nrclass) self.net.addOutputModule(outLayer) layers_connections = [] layers_connections.append(FullConnection(inLayer, hiddenLayers[0])) for i in range(self.hiddenlayerscount - 1): layers_connections.append( FullConnection(hiddenLayers[i - 1], hiddenLayers[i])) layers_connections.append( FullConnection(hiddenLayers[-1], outLayer)) for layers_connection in layers_connections: self.net.addConnection(layers_connection) self.net.sortModules() #training the network self.trainer = BackpropTrainer(self.net, self.ds) self.trainer.train() def PreProcessTestDate(self): self.testdata = preprocess_apply(self.testdata, self.missingvaluemethod, self.preprocessingmethods) def Predict(self): prediction = [] attributescount = len(self.testdata[0]) nrclass = len(set(self.testlabel)) dstraindata = ClassificationDataSet(attributescount, target=nrclass, nb_classes=nrclass, class_labels=list( set(self.testlabel))) for i in range(len(self.testdata)): dstraindata.appendLinked(self.testdata[i], self.testlabel[i]) dstraindata._convertToOneOfMany() out = self.net.activateOnDataset(dstraindata) prediction = out.argmax(axis=1) ''' for testrecord in self.testdata : out = self.net.activate(testrecord)[0] prediction.append(out) ''' self.result = [self.testlabel, prediction] def GetModel(self): return self.trainer
dataset.setField('target', y) # train the network trainer = RPropMinusTrainerMix(n, dataset=dataset, verbose=True, weightdecay=0.05) trainer.trainEpochs(200) # plot the density and other stuff p.subplot(2, 2, 3) dens = [] newx = np.arange(0.0, 1.0, 0.01) newx = newx.reshape(newx.size, 1) dataset.setField('input', newx) out = n.activateOnDataset(dataset) for pars in out: stds = pars[N_GAUSSIANS:N_GAUSSIANS * 2] means = pars[N_GAUSSIANS * 2:N_GAUSSIANS * 3] line = multigaussian(newx, means, stds) density = line[:, 0] * pars[0] for gaussian in range(1, N_GAUSSIANS): density += line[:, gaussian] * pars[gaussian] dens.append(density) newx = newx.flatten() dens = np.array(dens).transpose() p.contourf(newx, newx, dens, 30) p.title("cond. probab. dens.") p.subplot(221)
net.addOutputModule(outLayer) # # do the plumbing # in_to_hidden1 = FullConnection(inLayer,hiddenLayer1) hidden1_to_hidden2 = FullConnection(hiddenLayer1,hiddenLayer2) hidden2_to_out = FullConnection(hiddenLayer2,outLayer) # net.addConnection(in_to_hidden1) net.addConnection(hidden1_to_hidden2) net.addConnection(hidden2_to_out) net.sortModules() # # activate on the training data set # net.activateOnDataset(trndata) # # build a backpropagation trainer # trainer = BackpropTrainer(net, \ dataset=trndata, \ momentum=0.1, \ verbose=True, \ weightdecay=0.01) # # Generate a square grid of data points and put it into # a dataset, which we can then classify to get a nice # contour field for visualization...so the target values # for this data set aren't going to be used... #
if os.path.isfile(filetoopen): myfile = open('myparam2.txt', 'r') c = [] for line in myfile: c.append(float(line)) n._setParameters(c) else: myfile = open('myparam2.txt', 'w') for i in n.params: myfile.write(str(i) + '\n') myfile.close() #activate the neural networks act = SupervisedDataSet(1, 1) act.addSample((0.2, ), (0.880422606518061, )) n.activateOnDataset(act) #create the test DataSet x = numpy.arange(0.0, 1.0 + 0.01, 0.01) s = 0.5 + 0.4 * numpy.sin(2 * numpy.pi * x) tsts = SupervisedDataSet(1, 1) tsts.setField('input', x.reshape(len(x), 1)) tsts.setField('target', s.reshape(len(s), 1)) #read the train DataSet from file trndata = SupervisedDataSet.loadFromFile(os.path.join(os.getcwd(), 'trndata')) #create the trainer t = BackpropTrainer(n, learningrate=0.01, momentum=mom) #train the neural network from the train DataSet
# training print "beginning training" trainer = BackpropTrainer(n, train_ds, verbose=True) #trainer.trainUntilConvergence(maxEpochs=MAX_EPOCHS) trainer.trainUntilConvergence() output = open('nn' + h2 + '.pkl', 'wb') pickle.dump(n, output) output.close() ns[h2] = n # validation print "beginning validation" out = n.activateOnDataset(val_ds) actual = val_ds['target'] error = np.sqrt(np.sum((out - actual)**2) / len(val_ds)) print "RMSE:", error if min_error == -1 or error < min_error: min_error = error min_h = h2 stop = time.time() print "Time:", stop - start print "best number of h2 nodes:", min_h nbest = ns[min_h] else:
net.addOutputModule(outLayer) # # do the plumbing # in_to_hidden1 = FullConnection(inLayer, hiddenLayer1) hidden1_to_hidden2 = FullConnection(hiddenLayer1, hiddenLayer2) hidden2_to_out = FullConnection(hiddenLayer2, outLayer) # net.addConnection(in_to_hidden1) net.addConnection(hidden1_to_hidden2) net.addConnection(hidden2_to_out) net.sortModules() # # activate on the training data set # net.activateOnDataset(trndata) # # build a backpropagation trainer # trainer = BackpropTrainer(net, \ dataset=trndata, \ momentum=0.1, \ verbose=True, \ weightdecay=0.01) # # Generate a square grid of data points and put it into # a dataset, which we can then classify to get a nice # contour field for visualization...so the target values # for this data set aren't going to be used... #
dataset = SupervisedDataSet(1, 1) dataset.setField('input', x) dataset.setField('target', y) # train the network trainer = RPropMinusTrainerMix(n, dataset=dataset, verbose=True, weightdecay=0.05) trainer.trainEpochs(200) # plot the density and other stuff p.subplot(2, 2, 3) dens = [] newx = np.arange(0.0, 1.0, 0.01) newx = newx.reshape(newx.size, 1) dataset.setField('input', newx) out = n.activateOnDataset(dataset) for pars in out: stds = pars[N_GAUSSIANS:N_GAUSSIANS*2] means = pars[N_GAUSSIANS*2:N_GAUSSIANS*3] line = multigaussian(newx, means, stds) density = line[:,0] * pars[0] for gaussian in range(1, N_GAUSSIANS): density += line[:, gaussian] * pars[gaussian] dens.append(density) newx = newx.flatten() dens = np.array(dens).transpose() p.contourf(newx, newx, dens, 30) p.title("cond. probab. dens.") p.subplot(221)
class NeuralNet(regression): ''' #deprecated def __init__(self, inputDim, outputDim): \''' Initializes class parameters Input: \''' regression.__init__(self,inputDim, outputDim) #self.net = buildNetwork(inputDim, outputDim) self.net = FeedForwardNetwork() inLayer = LinearLayer(inputDim) hiddenLayer1 = TanhLayer(10) hiddenLayer2 = TanhLayer(10) outLayer = SigmoidLayer(outputDim) self.net.addInputModule(inLayer) self.net.addModule(hiddenLayer1) self.net.addModule(hiddenLayer2) self.net.addOutputModule(outLayer) in_to_hidden1 = FullConnection(inLayer, hiddenLayer1) hidden1_to_hidden2=FullConnection(hiddenLayer1, hiddenLayer2) hidden2_to_out = FullConnection(hiddenLayer2, outLayer) self.net.addConnection(in_to_hidden1) self.net.addConnection(hidden1_to_hidden2) self.net.addConnection(hidden2_to_out) self.net.sortModules() self.shape=self.net.params.shape self.ds = SupervisedDataSet(self.inputDimension, self.outputDimension) ''' def __init__(self, rs): regression.__init__(self,rs) self.learningRate=rs.learningRate self.momentum=rs.momentum self.net = FeedForwardNetwork() #input Layer inLayer = layersDict[rs.inputLayer](rs.inputDim) self.net.addInputModule(inLayer) #outputLayer outLayer = layersDict[rs.outputLayer](rs.outputDim) self.net.addOutputModule(outLayer) #no hidden Layer if(len(rs.hiddenLayers)==0): #connection between input and output Layer in_to_out = FullConnection(inLayer, outLayer) self.net.addConnection(in_to_out) if(rs.bias==True): bias= BiasUnit('bias') self.net.addModule(bias) bias_to_out = FullConnection(bias, outLayer) self.net.addConnection(bias_to_out) else : #hidden Layers hiddenLayers=[] for layer in rs.hiddenLayers: tmp=layersDict[layer[0]](layer[1]) self.net.addModule(tmp) hiddenLayers.append(tmp) #connection between input and first hidden Layer in_to_hidden=FullConnection(inLayer,hiddenLayers[0]) self.net.addConnection(in_to_hidden) #connection between hidden Layers i=0 for i in range(1,len(hiddenLayers)): hidden_to_hidden=FullConnection(hiddenLayers[i-1],hiddenLayers[i]) self.net.addConnection(hidden_to_hidden) #connection between last hidden Layer and output Layer hidden_to_out= FullConnection(hiddenLayers[i],outLayer) self.net.addConnection(hidden_to_out) if(rs.bias==True): bias=BiasUnit('bias') self.net.addModule(bias) for layer in hiddenLayers : bias_to_hidden = FullConnection(bias, layer) self.net.addConnection(bias_to_hidden) bias_to_out = FullConnection(bias, outLayer) self.net.addConnection(bias_to_out) #initilisation of weight self.net.sortModules() self.shape=self.net.params.shape self.net._setParameters(np.random.normal(0.0,0.1,self.shape)) self.ds = SupervisedDataSet(self.inputDimension, self.outputDimension) #print(self.net) def setTheta(self, theta): self.net._setParameters(theta.reshape(self.shape)) def getTheta(self): return self.net.params def load(self,thetaFile): ''' load wheight of the neural network from the thetafile ''' self.net._setParameters(np.loadtxt(thetaFile+".theta")) #print ("theta LOAD : ", self.net.params) return self.net.params def getTrainingData(self, inputData, outputData): ''' Verifies the validity of the given input and output data Data should be organized by columns Input: -inputdata, numpy N-D array -outputData, numpy N-D array ''' regression.getTrainingData(self,inputData, outputData) for i in range(self.numberOfSamples): self.ds.addSample(inputData[i],outputData[i]) def train(self): ''' Perform batch regression ''' trainer = BackpropTrainer(self.net, self.ds, learningrate=self.learningRate, momentum=self.momentum) minError=10 while(True): error=trainer.train() print(self.meanSquareError()) if(error<minError): minError=error self.saveTheta(self.rs.path+self.rs.thetaFile+".theta") #trainer.trainUntilConvergence(maxEpochs=10, verbose=True) #trainer.trainEpochs(10) def computeOutput(self, inputVal): ''' Returns the output depending on the given input and theta Input: -inputVal: numpy N-D array -theta: numpy N-D array Output: -fa_out: numpy N-D array, output approximated ''' assert(inputVal.shape[0]==self.inputDimension), "NeuralNet: Bad input format : " + str(inputVal.shape[0])+"/"+str(self.inputDimension) output=self.net.activate(inputVal) #print(output) return output def meanSquareError(self): output=self.net.activateOnDataset(self.ds) return np.mean((self.outputData - output)**2)
def run(): import scipy from scipy import linalg f = open("modelfitDatabase1.dat", "rb") import pickle dd = pickle.load(f) node = dd.children[13] rfs = node.children[0].data["ReversCorrelationRFs"] pred_act = numpy.array(node.children[0].data["ReversCorrelationPredictedActivities"]) pred_val_act = numpy.array(node.children[0].data["ReversCorrelationPredictedValidationActivities"]) training_set = node.data["training_set"] validation_set = node.data["validation_set"] training_inputs = node.data["training_inputs"] validation_inputs = node.data["validation_inputs"] ofs = contrib.modelfit.fit_sigmoids_to_of(numpy.mat(training_set), numpy.mat(pred_act)) pred_act_t = contrib.modelfit.apply_sigmoid_output_function(numpy.mat(pred_act), ofs) pred_val_act_t = contrib.modelfit.apply_sigmoid_output_function(numpy.mat(pred_val_act), ofs) (sx, sy) = numpy.shape(rfs[0]) print sx, sy n = FeedForwardNetwork() inLayer = LinearLayer(sx * sy) hiddenLayer = SigmoidLayer(4) outputLayer = SigmoidLayer(1) n.addInputModule(inLayer) n.addModule(hiddenLayer) n.addOutputModule(outputLayer) in_to_hidden = RBFConnection(sx, sy, inLayer, hiddenLayer) # in_to_hidden = FullConnection(inLayer, hiddenLayer) hidden_to_out = FullConnection(hiddenLayer, outputLayer) n.addConnection(in_to_hidden) n.addConnection(hidden_to_out) n.sortModules() gradientCheck(n) return from pybrain.datasets import SupervisedDataSet ds = SupervisedDataSet(sx * sy, 1) val = SupervisedDataSet(sx * sy, 1) for i in xrange(0, len(training_inputs)): ds.addSample(training_inputs[i], training_set[i, 0]) for i in xrange(0, len(validation_inputs)): val.addSample(validation_inputs[i], validation_set[i, 0]) tstdata, trndata = ds.splitWithProportion(0.1) from pybrain.supervised.trainers import BackpropTrainer trainer = BackpropTrainer(n, trndata, momentum=0.1, verbose=True, learningrate=0.002) training_set = numpy.array(numpy.mat(training_set)[:, 0]) validation_set = numpy.array(numpy.mat(validation_set)[:, 0]) pred_val_act_t = numpy.array(numpy.mat(pred_val_act_t)[:, 0]) out = n.activateOnDataset(val) (ranks, correct, pred) = contrib.modelfit.performIdentification(validation_set, out) print "Correct:", correct, "Mean rank:", numpy.mean(ranks), "MSE", numpy.mean(numpy.power(validation_set - out, 2)) print "Start training" for i in range(50): trnresult = percentError(trainer.testOnData(), trndata) tstresult = percentError(trainer.testOnData(dataset=tstdata), tstdata) print "epoch: %4d" % trainer.totalepochs, " train error: %5.2f%%" % trnresult, " test error: %5.2f%%" % tstresult trainer.trainEpochs(1) out = n.activateOnDataset(val) (ranks, correct, pred) = contrib.modelfit.performIdentification(validation_set, out) print "Correct:", correct, "Mean rank:", numpy.mean(ranks), "MSE", numpy.mean( numpy.power(validation_set - out, 2) ) out = n.activateOnDataset(val) print numpy.shape(out) print numpy.shape(validation_set) (ranks, correct, pred) = contrib.modelfit.performIdentification(validation_set, out) print "Correct:", correct, "Mean rank:", numpy.mean(ranks), "MSE", numpy.mean(numpy.power(validation_set - out, 2)) (ranks, correct, pred) = contrib.modelfit.performIdentification(validation_set, pred_val_act_t) print "Correct:", correct, "Mean rank:", numpy.mean(ranks), "MSE", numpy.mean( numpy.power(validation_set - pred_val_act_t, 2) ) return n
)) # Инициализируем веса сети для получения воспроизводимого результата net._setParameters(init_params) #%% np.random.seed(0) # Модуль настройки параметров pybrain использует модуль random; зафиксируем seed для получения воспроизводимого результата trainer = BackpropTrainer( net, dataset=ds_train) # Инициализируем модуль оптимизации err_train, err_val = trainer.trainUntilConvergence(maxEpochs=MAX_EPOCHS) line_train = plt.plot(err_train, 'b', err_val, 'r') # Построение графика xlab = plt.xlabel('Iterations') ylab = plt.ylabel('Error') #%% #ROC - кривые - порог grance = 0.5 res_train = net.activateOnDataset( ds_train) # Подсчет результата на обучающей выборке res_train_bin = [] for i in res_train: if i > grance: res_train_bin.append(1) else: res_train_bin.append(0) print('Error on train: ', percentError(res_train_bin, ds_train['target'])) # Подсчет ошибки res_test = net.activateOnDataset( ds_test) # Подсчет результата на тестовой выборке res_test_bin = [] for i in res_test: if i > grance: res_test_bin.append(1)