def train(input, hidden, output, dataset): """ Method to build and train a neural network with backpropagation until it converges :param input: input nodes for network (1) :param hidden: hidden nodes for network :param output: output nodes for network (1) :param dataset: SupervisedDataSet, maps input->output: 1->1, 2->2 ... 8->8 :return: the trained neural network """ net = buildNetwork(input, hidden, output, hiddenclass=TanhLayer) trainer = BackpropTrainer(net, dataset, learningrate=0.01) trainer.trainUntilConvergence(verbose=False, validationProportion=0.15, maxEpochs=1000, continueEpochs=10) return net
def get_trained_ann(dataset, ann=None, test_train_prop=0.25, max_epochs=50): tstdata, trndata = dataset.splitWithProportion(test_train_prop) trndata._convertToOneOfMany() tstdata._convertToOneOfMany() if not ann: ann = build_ann(trndata.indim, trndata.outdim) # ann = build_exp_ann(trndata.indim, trndata.outdim) # trainer = RPropMinusTrainer(ann) trainer = BackpropTrainer(ann, dataset=trndata,learningrate=0.01, momentum=0.5, verbose=True) trnresult = tstresult = 0 # for i in range(10): trainer.trainUntilConvergence(maxEpochs=max_epochs, verbose=True) trnresult = percentError( trainer.testOnClassData(), trndata['class'] ) tstresult = percentError( trainer.testOnClassData(dataset=tstdata ), tstdata['class'] ) # print trnresult, tstresult return ann, trnresult, tstresult
def NetworkTrain(trainDataSet, mnetwork=NetworkBuild(), file='NetworkDump.pkl',maxEpochs=100): mnetwork = NetworkBuild(new = True) assert len(mnetwork[0].inmodules) == len(mnetwork[1].keys()) print('DEBUG') #print(trainDataSet) print("lens " + str(len(trainDataSet[0][0])) + " " + str(len(mnetwork[0].inmodules))) # 定义数据集的格式 DS = SupervisedDataSet(len(trainDataSet[0][0]), len(trainDataSet[0][1])) for itrainDataSet in trainDataSet: indata = itrainDataSet[0] outdata = itrainDataSet[1] DS.addSample(indata, outdata) # 如果要获得里面的输入/输出时,可以用 # 如果要把数据集切分成训练集和测试集,可以用下面的语句,训练集:测试集=8:2 # 为了方便之后的调用,可以把输入和输出拎出来 # 训练器采用BP算法 # verbose = True即训练时会把Total error打印出来,库里默认训练集和验证集的比例为4:1,可以在括号里更改 mnetwork[0].sortModules() trainer = BackpropTrainer(mnetwork[0], DS, verbose=True, learningrate=0.01) # 0.0575 # maxEpochs即你需要的最大收敛迭代次数,这里采用的方法是训练至收敛,我一般设为1000 trainer.trainUntilConvergence(maxEpochs=maxEpochs) ''' for mod in mnetwork[0].modules: print "Module:", mod.name if mod.paramdim > 0: print "--parameters:", mod.params for conn in mnetwork[0].connections[mod]: print "-connection to", conn.outmod.name if conn.paramdim > 0: print "- parameters", conn.params if hasattr(mnetwork[0], "recurrentConns"): print "Recurrent connections" for conn in mnetwork[0].recurrentConns: print "-", conn.inmod.name, " to", conn.outmod.name if conn.paramdim > 0: print "- parameters", conn.params ''' pickle.dump(mnetwork, open(file, 'wb')) return mnetwork
def train(self, x, y, class_number=-1): self.__class_num = max(np.unique(y).size, class_number) if max(y) == self.__class_num: self.__class_zero_indexing = False y = np.array([i - 1 for i in y]) DS = ClassificationDataSet(x.shape[1], nb_classes=self.__class_num) DS.setField('input', x) DS.setField('target', y.reshape(y.size, 1)) DS._convertToOneOfMany() hidden_num = (DS.indim + DS.outdim) / 2 self.__pybrain_bpnn = buildNetwork(DS.indim, hidden_num, DS.outdim, bias=True, hiddenclass=SigmoidLayer, outclass=SoftmaxLayer) trainer = BackpropTrainer(self.__pybrain_bpnn, dataset=DS, learningrate=0.07, lrdecay=1.0, momentum=0.6) trainer.trainUntilConvergence(DS, maxEpochs=30)
def train(network_file, input_length, output_length, training_data_file, learning_rate, momentum, stop_on_convergence, epochs, classify): n = get_network(network_file) if classify: ds = ClassificationDataSet(int(input_length), int(output_length) * 2) ds._convertToOneOfMany() else: ds = SupervisedDataSet(int(input_length), int(output_length)) training_data = get_training_data(training_data_file) NetworkManager.last_training_set_length = 0 for line in training_data: data = [float(x) for x in line.strip().split(',') if x != ''] input_data = tuple(data[:(int(input_length))]) output_data = tuple(data[(int(input_length)):]) ds.addSample(input_data, output_data) NetworkManager.last_training_set_length += 1 t = BackpropTrainer(n, learningrate=learning_rate, momentum=momentum, verbose=True) print "training network " + network_storage_path + network_file if stop_on_convergence: t.trainUntilConvergence(ds, epochs) else: if classify: t.trainOnDataset(ds['class'], epochs) else: t.trainOnDataset(ds, epochs) error = t.testOnData() print "training done" if not math.isnan(error): save_network(n, network_file) else: print "error occured, network not saved" print "network saved" return error
def get_trained_ann(dataset, ann=None, test_train_prop=0.25, max_epochs=50): tstdata, trndata = dataset.splitWithProportion(test_train_prop) trndata._convertToOneOfMany() tstdata._convertToOneOfMany() if not ann: ann = build_ann(trndata.indim, trndata.outdim) # ann = build_exp_ann(trndata.indim, trndata.outdim) # trainer = RPropMinusTrainer(ann) trainer = BackpropTrainer(ann, dataset=trndata, learningrate=0.01, momentum=0.5, verbose=True) trnresult = tstresult = 0 # for i in range(10): trainer.trainUntilConvergence(maxEpochs=max_epochs, verbose=True) trnresult = percentError(trainer.testOnClassData(), trndata['class']) tstresult = percentError(trainer.testOnClassData(dataset=tstdata), tstdata['class']) # print trnresult, tstresult return ann, trnresult, tstresult
def main(): # carregando o dataset do iris dataset = montaDataset() # carregando os datasets temporários # separando 60% dos dados # para treino e 30 % para teste datasetTreinoTemporario, dadosRepartidosTemporario = dataset.splitWithProportion( 0.6) # carregando os datasets temporários # separando os dados repartidos # em 50 % para teste e # os outros 50 % para validação datasetTesteTemporario, datasetValidacaoTemporario = dadosRepartidosTemporario.splitWithProportion( 0.5) # montandos os datasets finais datasetTreino = montaDatasetConvertido(datasetTreinoTemporario) datasetTeste = montaDatasetConvertido(datasetTesteTemporario) datasetValidacao = montaDatasetConvertido(datasetValidacaoTemporario) # definindo a estrutura da rede adpatadas ao SoftmaxLayer # com dimensão 4 de entrada # com 20 neurônios na 1 camada intermediária # com 2 camadas # com dimensão 3 na saída # terá como saída adaptada à classe SoftmaxLayer network = buildNetwork(4, 20, 2, 3, outclass=SoftmaxLayer) # convertendo os dados para o objeto ConvertToOneOfMany datasetTreino._convertToOneOfMany() datasetTeste._convertToOneOfMany() datasetValidacao._convertToOneOfMany() # criando a rede neural treinada # passando a estrutura da rede neural # dataset para treino neuralNetwork = BackpropTrainer(network, dataset=datasetTreino, learningrate=0.02, momentum=0.14, verbose=True) # treinando a rede até ela convergir # e semparando os erros em um array de erro do treino # e um array dos erros da validação errosTreino, validacaoErros = neuralNetwork.trainUntilConvergence( dataset=datasetTreino, maxEpochs=1000) # criando a camada externa de teste outTest = network.activateOnDataset(datasetTeste).argmax(axis=1) print(5 * "-" + " Precisão de teste de treinamento " + 5 * "-" + "\n") print("Precisão no teste : {} %".format( 100 - percentError(outTest, datasetTeste["class"]))) # criando a camada externa de validação outVal = network.activateOnDataset(datasetValidacao).argmax(axis=1) print("\n" + 5 * "-" + " Precisão de teste de treinamento " + 5 * "-" + "\n") print("Precisão na validação : {} %".format( 100 - percentError(outVal, datasetTeste["class"]))) # fazendo o plot gráfico dos erros plt.plot(errosTreino, "b", validacaoErros, "r") plt.show()
for conn in net.connections[mod]: print >> net_output_file, "-connection to", conn.outmod.name print "-connection to", conn.outmod.name if conn.paramdim > 0: print >> net_output_file, "- parameters", conn.params print "- parameters", conn.params if hasattr(net, "recurrentConns"): print "Recurrent connections" for conn in net.connections[mod]: print >> net_output_file, "-", conn.inmod.name, " to", conn.outmod.name print "-", conn.inmod.name, " to", conn.outmod.name if conn.paramdim > 0: print >> net_output_file, "- parameters", conn.params print "- parameters", conn.params trainer_message = trainer.trainUntilConvergence(maxEpochs=1000) print >> net_output_file, "train-errors:" net_output_file.write('[\t') for i in range(len(trainer_message[0])): net_output_file.write(str(trainer_message[0][i])) if i != len(trainer_message[0]) - 1: net_output_file.write('\t') net_output_file.write(']\n') print >> net_output_file, "valid-errors:" net_output_file.write('[\t') for i in range(len(trainer_message[1])): net_output_file.write(str(trainer_message[1][i])) if i != len(trainer_message[1]) - 1: net_output_file.write('\t')
class NeuralNetwork(object): """ The neural network class does all the heavy lifting to incorporate pybrain neural networks into the NowTrade ecosystem. """ def __init__(self, train_data, prediction_data, network_type=FEED_FORWARD_NETWORK, network_dataset_type=SUPERVISED_DATASET, trainer_type=BACKPROP_TRAINER): self.train_data = train_data self.prediction_data = prediction_data self.network_type = network_type self.network_dataset_type = network_dataset_type self.trainer_type = trainer_type self.network = None self.network_dataset = None self.dataset = None self.trainer = None self.trained_iterations = 0 self.momentum = None self.learning_rate = None self.hidden_layers = None self.prediction_window = None self.logger = logger.Logger(self.__class__.__name__) self.logger.info('train_data: %s prediction_data: %s, network_type: %s, \ network_dataset_type: %s, trainer_type: %s' %(train_data, prediction_data, network_type, \ network_dataset_type, trainer_type)) def save(self): """ Returns the pickled trained/tested neural network as a string. """ return cPickle.dumps(self) def save_to_file(self, filename): """ Saves a neural network to file for later use. Look into pybrain.datasets.supervised.SupervisedDataSet.saveToFile() http://pybrain.org/docs/api/datasets/superviseddataset.html """ file_handler = open(filename, 'wb') cPickle.dump(self, file_handler) file_handler.close() def build_network(self, dataset, new=True, **kwargs): """ Builds a neural network using the dataset provided. Expected keyword args: - 'hidden_layers' - 'prediction_window' - 'learning_rate' - 'momentum' """ self.hidden_layers = kwargs.get('hidden_layers', 3) self.prediction_window = kwargs.get('prediction_window', 1) self.learning_rate = kwargs.get('learning_rate', 0.1) self.momentum = kwargs.get('momentum', 0.01) if not new: self.network.sorted = False self.network.sortModules() if self.network_dataset_type == SUPERVISED_DATASET: self.ready_supervised_dataset(dataset) else: raise InvalidNetworkDatasetType() else: if self.network_type == FEED_FORWARD_NETWORK: self.network = buildNetwork(len(self.train_data), self.hidden_layers, 1) else: raise InvalidNetworkType() if self.network_dataset_type == SUPERVISED_DATASET: self.ready_supervised_dataset(dataset) else: raise InvalidNetworkDatasetType() if self.trainer_type == BACKPROP_TRAINER: self.trainer = BackpropTrainer(self.network, learningrate=self.learning_rate, momentum=self.momentum, verbose=True) self.trainer.setData(self.network_dataset) else: raise InvalidTrainerType() def ready_supervised_dataset(self, dataset): """ Ready the supervised dataset for training. @TODO: Need to randomize the data being fed to the network. See randomBatches() here: http://pybrain.org/docs/api/datasets/superviseddataset.html """ self.network_dataset = SupervisedDataSet(len(self.train_data), 1) # Currently only supports log function for normalizing data training_values = np.log(dataset.data_frame[self.train_data]) results = np.log(dataset.data_frame[self.prediction_data].shift( -self.prediction_window)) training_values['PREDICTION_%s' % self.prediction_data[0]] = results training_values = training_values.dropna() for _, row_data in enumerate(training_values.iterrows()): _, data = row_data sample = list(data[:-1]) result = [data[-1]] self.network_dataset.addSample(sample, result) def train(self, cycles=1): """ Trains the network the number of iteration specified in the cycles parameter. """ for _ in range(cycles): res = self.trainer.train() self.trained_iterations += 1 return res def train_until_convergence(self, max_cycles=1000, continue_cycles=10, validation_proportion=0.25): """ Wrapper around the pybrain BackpropTrainer trainUntilConvergence method. @see: http://pybrain.org/docs/api/supervised/trainers.html """ self.trainer = \ self.trainer.trainUntilConvergence(maxEpochs=max_cycles, continueEpochs=continue_cycles, validationProportion=validation_proportion) def _activate(self, data): """ Activates the network using the data specified. Returns the network's prediction. """ return self.network.activate(data)[0] def activate_all(self, data_frame): """ Activates the network for all values in the dataframe specified. """ dataframe = np.log(data_frame[self.train_data]) res = [] for _, row_data in enumerate(dataframe.iterrows()): _, data = row_data sample = list(data) res.append(self._activate(sample)) return np.exp(res)
for conn in net.connections[mod]: print >> net_output_file, "-connection to", conn.outmod.name print "-connection to", conn.outmod.name if conn.paramdim > 0: print >> net_output_file, "- parameters", conn.params print "- parameters", conn.params if hasattr(net, "recurrentConns"): print "Recurrent connections" for conn in net.connections[mod]: print >> net_output_file , "-", conn.inmod.name, " to", conn.outmod.name print "-", conn.inmod.name, " to", conn.outmod.name if conn.paramdim > 0: print >> net_output_file, "- parameters", conn.params print "- parameters", conn.params trainer_message = trainer.trainUntilConvergence(maxEpochs=1000) print >> net_output_file, "train-errors:" net_output_file.write('[\t') for i in range(len(trainer_message[0])): net_output_file.write(str(trainer_message[0][i])) if i!=len(trainer_message[0])-1: net_output_file.write('\t') net_output_file.write(']\n') print >> net_output_file, "valid-errors:" net_output_file.write('[\t') for i in range(len(trainer_message[1])): net_output_file.write(str(trainer_message[1][i])) if i!=len(trainer_message[1])-1: net_output_file.write('\t')
print 'build set' alldata = ClassificationDataSet(dim, 1, nb_classes=2) (data,label,items) = BinReader.readData(ur'F:\AliRecommendHomeworkData\1212新版\train15_17.expand.samp.norm.bin') #(train,label,data) = BinReader.readData(r'C:\data\small\norm\train1217.bin') for i in range(len(data)): alldata.addSample(data[i],label[i]) tstdata, trndata = alldata.splitWithProportion(0.25) trainer = BackpropTrainer(n,trndata,momentum=0.1,verbose=True,weightdecay=0.01) print 'start' #trainer.trainEpochs(1) trainer.trainUntilConvergence(maxEpochs=2) trnresult = percentError(trainer.testOnClassData(),trndata['class']) tstresult = percentError(trainer.testOnClassData(dataset=tstdata), tstdata['class']) print "epoch: %4d" % trainer.totalepochs, \ " train error: %5.2f%%" % trnresult, \ " test error: %5.2f%%" % tstresult print 'get result' #trainer.trainUntilConvergence() #out = ClassificationDataSet(37,1) #(test,label,items) = BinReader.readData(r'C:\data\homework\1218t5w.bin') ##(test,label,data) = BinReader.readData(r'C:\data\small\norm\test1218.bin') #for i in range(len(test)):
trainingSet = SupervisedDataSet(len(championdictionary),1) #Takes each game and turns it into a vector. -1 is stored if the champion is on the opposing team, 1 if the champion is on the player's team #and 0 if it wasn't played. The vector is then fed into the Neural Network's Training Set print "Adding Games to NN" for game in aramdata.readlines(): aramgame = json.loads(game) teammates,opponents, gameWin = DatabaseActions.GetResults(aramgame) #writes a vector of which champions were on which team followed by the result gamevector = [0]*len(championdictionary) for champion in teammates: gamevector[championdictionary[str(champion)]["Id"]-1] = 1 for champion in opponents: gamevector[championdictionary[str(champion)]["Id"]-1] = -1 #Feeds that result into our Neural Network's training set trainingSet.appendLinked(gamevector,int(gameWin)) #Creates a Backpropagation trainer and proceeds to train on our set. This step can take a while due to the volume of our data. print "Training NN" trainer = BackpropTrainer(predictionNet,trainingSet) trainer.trainUntilConvergence(dataset = trainingSet, maxEpochs = 300, verbose = True, continueEpochs = 10, validationProportion=0.1) #Saves our trained Network by exporting it to an XML file NetworkWriter.writeToFile(predictionNet, "TrainedNetwork.xml") aramdata.close()
#add entries to the dataset for entry in result: ds.addSample(entry[0], (entry[1])) leftDs, rightDs = ds.splitWithProportion(training_set_percentage) if (use_stored_weights == False): print("successs rate over all names before training:") print(validate_dataset(result, net)) print() #train the network print("starting to train the network") trainer = BackpropTrainer(net, leftDs) trainer.trainUntilConvergence(maxEpochs=max_epochs) #store learned parameters to file print("validate against the validation dataset") print(validate_dataset(rightDs, net)) print() with io.open('data/weights.json', 'w', encoding='utf8') as f: f.write(json.dumps(net.params.tolist())) else: with io.open('data/weights.json') as f: weights = f.read() weights = json.loads(weights) net._setParameters(weights)
def base_experiment(): (eval_dataset, eval_costset) = DomainFnApprox.make_evaluation_datasets() random_train_dataset = SupervisedDataSet(2, 1) random_train_costset = SupervisedDataSet(2, 1) for i in range(RANDOM_TRAINING_SAMPLES): x = random.uniform(X_MIN, X_MAX) y = random.uniform(Y_MIN, Y_MAX) z = FN(x, y) z_cost = COST_FN(x, y) random_train_dataset.addSample([x, y], [z]) random_train_costset.addSample([x, y], [z_cost]) value_network = buildNetwork(2, 80, 20, 1, hiddenclass=SigmoidLayer, bias=True) value_trainer = BackpropTrainer(value_network, learningrate=LEARNING_RATE, momentum=MOMENTUM, verbose=True) print 'Value Network Topology:' print value_network cost_network = buildNetwork(2, 80, 20, 1, hiddenclass=SigmoidLayer, bias=True) cost_trainer = BackpropTrainer(cost_network, learningrate=LEARNING_RATE, momentum=MOMENTUM, verbose=True) # test_derivatives(value_network, [1, 1]) # test_derivatives(cost_network, [1, 1]) print 'Value MSE before: %.4f' % value_trainer.testOnData(eval_dataset) value_trainer.trainUntilConvergence(random_train_dataset, continueEpochs=6, maxEpochs=MAX_EPOCHS) # value_trainer.trainOnDataset(random_train_dataset, 1000) print 'Value MSE after: %.4f' % value_trainer.testOnData(eval_dataset) print 'Cost MSE before: %.4f' % cost_trainer.testOnData(eval_costset) cost_trainer.trainUntilConvergence(random_train_costset, continueEpochs=6, maxEpochs=MAX_EPOCHS) # cost_trainer.trainOnDataset(random_train_costset, 1000) print 'Cost MSE after: %.4f' % cost_trainer.testOnData(eval_costset) # test_derivatives(value_network, [1, 1]) # test_derivatives(cost_network, [1, 1]) f_value = open('../data/learnedvalue.txt', 'w') f_cost = open('../data/learnedcost.txt', 'w') unit = (X_MAX - X_MIN) / (EVAL_SAMPLES_AXIS - 1) for i in range(EVAL_SAMPLES_AXIS): for j in range(EVAL_SAMPLES_AXIS): x = X_MIN + i * unit y = Y_MIN + j * unit z = value_network.activate([x, y]) z_cost = cost_network.activate([x, y]) f_value.write('%f %f %f\n' % (x, y, z[0])) f_cost.write('%f %f %f\n' % (x, y, z_cost[0])) f_value.close() f_cost.close()
hidden = int(argv[1]) # X # Expects one dimensional input and target output ds = SupervisedDataSet(1, 1) for x in range(1, 9): ds.addSample(x, x) # 1 input, X hidden, 1 output network = buildNetwork(1, hidden, 1, hiddenclass=TanhLayer) # Init BackpropTrainer trainer = BackpropTrainer(network, dataset=ds) # Train until convergence trainer.trainUntilConvergence(verbose=False, validationData=0.15, maxEpochs=1000, continueEpochs=10) # Activating the network on different integers such as the inputs in the data-set print("// Hidden nodes: {}".format(hidden)) for x in range(1, 9): print("{} --> {}".format(x, network.activate([x])[0])) # Activating with decimal inputs outside of dataset print("\nDecimal input outside dataset:") for x in range(100, 109): print("{} --> {}".format(x, network.activate([x * 0.1])[0])) # Activating with negative numbers outside of data-set print("\nNegative input:") for x in range(1, 9):
class PyBrainNetwork(ANNWrapper): def __init__(self, hidden_neurons=None, output_class="softmax", learning_rate=0.01, lr_decay=1.0, momentum=0.0, weight_decay=0.0, max_epochs=None, continue_epochs=10, validation_percent=0.25, random_state=None): super(PyBrainNetwork, self).__init__() self.hidden_neurons = hidden_neurons self.output_class = output_class self.learning_rate = learning_rate self.lr_decay = lr_decay self.momentum = momentum self.weight_decay = weight_decay self.max_epochs = max_epochs self.continue_epochs = continue_epochs self.validation_percent = validation_percent self.random_state = random_state self.network_ = None self.trainer_ = None def fit(self, inp, y, sample_weight=None): self.classes_, y = numpy.unique(y, return_inverse=True) self.n_classes_ = len(self.classes_) n_features = inp.shape[1] random_state = check_random_state(self.random_state) # We need to build an ImportanceDataSet from inp, y and sample_weight dataset = ImportanceDataSet(n_features, self.n_classes_) if sample_weight is None: sample_weight = numpy.ones(len(y)) for x, label_pos, weight in izip(inp, y, sample_weight): target = numpy.zeros(self.n_classes_) target[label_pos] = 1 weight = weight * numpy.ones(self.n_classes_) dataset.newSequence() dataset.addSample(x, target, weight) if self.hidden_neurons is None: hidden_neurons = (n_features + self.n_classes_)/2 else: hidden_neurons = self.hidden_neurons self.network_ = buildNetwork( n_features, hidden_neurons, self.n_classes_, outclass=self._get_output_class() ) # Set the initial parameters in a repeatable way net_params = random_state.random_sample(self.network_.paramdim) self.network_.params[:] = net_params self.trainer_ = BackpropTrainer( self.network_, dataset=dataset, learningrate=self.learning_rate, lrdecay=self.lr_decay, momentum=self.momentum, weightdecay=self.weight_decay ) self.trainer_.trainUntilConvergence( random_state, maxEpochs=self.max_epochs, continueEpochs=self.continue_epochs, validationProportion=self.validation_percent ) return self def activate(self, x): return self.network_.activate(x) def _get_output_class(self): if self.output_class == "softmax": return SoftmaxLayer elif self.output_class == "tanh": return TanhLayer elif self.output_class == "sigmoid": return SigmoidLayer elif self.output_class == "linear": return LinearLayer else: raise ValueError( "output_class can be: softmax, tanh, sigmoid, linear" )
class NeuralNetwork(object): '''Neural network wrapper for the pybrain implementation ''' def __init__(self): self.path = os.path.dirname( os.path.abspath(__file__)) + "/../../../data/" self.net = None self.trainer = None def createNew(self, nInputs, nHiddenLayers, nOutput, bias): '''builds a new neural network :param int nInputs: the number of input nodes :param int nHiddenLayers: the number of hidden layers :parma int nOutputs: the number of output nodes :param bool bias: if True an bias node will be added :return: instance of a new neural network :rtype: NeuralNetwork ''' self.net = buildNetwork(nInputs, nHiddenLayers, nOutput, bias=bias, hiddenclass=TanhLayer) return self def train(self, dataset, maxEpochs=10, learningrate=0.01, momentum=0.99, continueEpochs=10, validationProportion=0.25): '''trains a network with the given dataset :param SupervisedDataSet dataset: the training dataset :param int maxEpochs: max number of iterations to train the network :parma float learningrate: helps to :param float momentum: helps out of local minima while training, to get better results ''' self.trainer = BackpropTrainer(self.net, learningrate=learningrate, momentum=momentum) self.trainer.trainOnDataset(dataset, maxEpochs) def trainConvergence(self, dataset, maxEpochs=10, learningrate=0.01, momentum=0.99, continueEpochs=10, validationProportion=0.25): '''trains a network with the given dataset nutil it converges :param SupervisedDataSet dataset: the training dataset :param int maxEpochs: max number of iterations to train the network :parma float learningrate: helps to :param float momentum: helps out of local minima while training, to get better results ''' self.trainer = BackpropTrainer(self.net, learningrate=learningrate, momentum=momentum) self.trainer.trainUntilConvergence(dataset, maxEpochs, False, continueEpochs, validationProportion) def test(self, data=None, verbose=False): if not self.trainer: raise ValueError( "call train() first, to create a valid trainer object") return self.trainer.testOnData(data, verbose) def activate(self, value, rnd=False): inpt = self.net.activate(value)[0] if rnd: return self._clazz(inpt) return inpt def _clazz(self, inpt): clazz = round(inpt) if (clazz < 0): return 0 if (clazz > 1): return 1 return int(clazz) def _createFilePath(self, filePath, defaultPath=True): if defaultPath: return self.path + filePath + FILE_EXTENSION return filePath + FILE_EXTENSION def save(self, filePath, defaultPath=True): '''saves the neural network :param string filePath: filepath for the to be saved network ''' with open(self._createFilePath(filePath, defaultPath), 'w') as f: pickle.dump(self.net, f) def load(self, filePath, defaultPath=True): '''loades the neural network :param string filePath: filepath for the to be loaded network :return: instance of a saved neural network :rtype: NeuralNetwork ''' with open(self._createFilePath(filePath, defaultPath), 'r') as f: self.net = pickle.load(f) return self def __repr__(self): return "%s\n%s" % (self.__class__.__name__, str(self.net))
class NeuralNetwork(object): """ The neural network class does all the heavy lifting to incorporate pybrain neural networks into the NowTrade ecosystem. """ def __init__(self, train_data, prediction_data, network_type=FEED_FORWARD_NETWORK, network_dataset_type=SUPERVISED_DATASET, trainer_type=BACKPROP_TRAINER): self.train_data = train_data self.prediction_data = prediction_data self.network_type = network_type self.network_dataset_type = network_dataset_type self.trainer_type = trainer_type self.network = None self.network_dataset = None self.dataset = None self.trainer = None self.trained_iterations = 0 self.momentum = None self.learning_rate = None self.hidden_layers = None self.prediction_window = None self.logger = logger.Logger(self.__class__.__name__) self.logger.info('train_data: %s prediction_data: %s, network_type: %s, \ network_dataset_type: %s, trainer_type: %s' %(train_data, prediction_data, network_type, \ network_dataset_type, trainer_type)) def save(self): """ Returns the pickled trained/tested neural network as a string. """ return cPickle.dumps(self) def save_to_file(self, filename): """ Saves a neural network to file for later use. Look into pybrain.datasets.supervised.SupervisedDataSet.saveToFile() http://pybrain.org/docs/api/datasets/superviseddataset.html """ file_handler = open(filename, 'wb') cPickle.dump(self, file_handler) file_handler.close() def build_network(self, dataset, new=True, **kwargs): """ Builds a neural network using the dataset provided. Expected keyword args: - 'hidden_layers' - 'prediction_window' - 'learning_rate' - 'momentum' """ self.hidden_layers = kwargs.get('hidden_layers', 3) self.prediction_window = kwargs.get('prediction_window', 1) self.learning_rate = kwargs.get('learning_rate', 0.1) self.momentum = kwargs.get('momentum', 0.01) if not new: self.network.sorted = False self.network.sortModules() if self.network_dataset_type == SUPERVISED_DATASET: self.ready_supervised_dataset(dataset) else: raise InvalidNetworkDatasetType() else: if self.network_type == FEED_FORWARD_NETWORK: self.network = buildNetwork(len(self.train_data), self.hidden_layers, 1) else: raise InvalidNetworkType() if self.network_dataset_type == SUPERVISED_DATASET: self.ready_supervised_dataset(dataset) else: raise InvalidNetworkDatasetType() if self.trainer_type == BACKPROP_TRAINER: self.trainer = BackpropTrainer(self.network, learningrate=self.learning_rate, momentum=self.momentum, verbose=True) self.trainer.setData(self.network_dataset) else: raise InvalidTrainerType() def ready_supervised_dataset(self, dataset): """ Ready the supervised dataset for training. @TODO: Need to randomize the data being fed to the network. See randomBatches() here: http://pybrain.org/docs/api/datasets/superviseddataset.html """ self.network_dataset = SupervisedDataSet(len(self.train_data), 1) # Currently only supports log function for normalizing data training_values = np.log(dataset.data_frame[self.train_data]) results = np.log(dataset.data_frame[self.prediction_data].shift(-self.prediction_window)) training_values['PREDICTION_%s' %self.prediction_data[0]] = results training_values = training_values.dropna() for _, row_data in enumerate(training_values.iterrows()): _, data = row_data sample = list(data[:-1]) result = [data[-1]] self.network_dataset.addSample(sample, result) def train(self, cycles=1): """ Trains the network the number of iteration specified in the cycles parameter. """ for _ in range(cycles): res = self.trainer.train() self.trained_iterations += 1 return res def train_until_convergence(self, max_cycles=1000, continue_cycles=10, validation_proportion=0.25): """ Wrapper around the pybrain BackpropTrainer trainUntilConvergence method. @see: http://pybrain.org/docs/api/supervised/trainers.html """ self.trainer = \ self.trainer.trainUntilConvergence(maxEpochs=max_cycles, continueEpochs=continue_cycles, validationProportion=validation_proportion) def _activate(self, data): """ Activates the network using the data specified. Returns the network's prediction. """ return self.network.activate(data)[0] def activate_all(self, data_frame): """ Activates the network for all values in the dataframe specified. """ dataframe = np.log(data_frame[self.train_data]) res = [] for _, row_data in enumerate(dataframe.iterrows()): _, data = row_data sample = list(data) res.append(self._activate(sample)) return np.exp(res)
return m, m_min, m_max '''training dataset with random phase screens (2018.01.17)''' mx = np.abs(trnslos_all).max() ntrnslos_all = trnslos_all / mx n_frame = 20 norm_inp = ntrnslos_all[:, :n_frame * 72] trnds = SupervisedDataSet(72 * n_frame, 1) #norm_inp, trnslos_min, trnslos_max = normalise(trnslos) trnds.setField('input', norm_inp) trn_tar = np.empty([6000, 1]) trn_tar[:, 0] = np.arange(5, 11).repeat(1000) / 15 trnds.setField('target', trn_tar) '''learning process''' net = buildNetwork(72 * n_frame, 1000, 1, hiddenclass=TanhLayer) lr = 0.001 momentum = 0 lrdecay = 1 wdecay = 0 t = BackpropTrainer(net, trnds, learningrate=lr, lrdecay=lrdecay, momentum=momentum, verbose=True, batchlearning=False, weightdecay=wdecay) trnerr, vderr, trnData, vdData = t.trainUntilConvergence( maxEpochs=30, validationProportion=0.1)
print value_network cost_network = buildNetwork(1, 40, 20, 1, hiddenclass=SigmoidLayer, bias=True) cost_trainer = BackpropTrainer(cost_network, learningrate=0.01, momentum=0.00, verbose=True) print 'Value MSE before: %.4f' % value_trainer.testOnData(eval_dataset) value_trainer.trainUntilConvergence(dataset, continueEpochs=6, maxEpochs=500) # value_trainer.trainOnDataset(dataset, 1000) print 'Value MSE after: %.4f' % value_trainer.testOnData(eval_dataset) print 'Cost MSE before: %.4f' % cost_trainer.testOnData(eval_costset) cost_trainer.trainUntilConvergence(costset, continueEpochs=6, maxEpochs=500) # cost_trainer.trainOnDataset(costset, 1000) print 'Cost MSE after: %.4f' % cost_trainer.testOnData(eval_costset) # print cost_network.params f_value = open('../data2d/valueplot2ddata.txt', 'w') f_cost = open('../data2d/costplot2ddata.txt', 'w') plot_dataset = SupervisedDataSet(2, 1)
class NeuralNetwork(object): '''Neural network wrapper for the pybrain implementation ''' def __init__(self): self.path = os.path.dirname(os.path.abspath(__file__)) + "/../../data/" self.net = None self.trainer = None def createNew(self, nInputs, nHiddenLayers, nOutput, bias): '''builds a new neural network :param int nInputs: the number of input nodes :param int nHiddenLayers: the number of hidden layers :parma int nOutputs: the number of output nodes :param bool bias: if True an bias node will be added :return: instance of a new neural network :rtype: NeuralNetwork ''' self.net = buildNetwork(nInputs, nHiddenLayers, nOutput, bias=bias, hiddenclass=TanhLayer) return self def train(self, dataset, maxEpochs = 10, learningrate = 0.01, momentum = 0.99, continueEpochs=10, validationProportion=0.25): '''trains a network with the given dataset :param SupervisedDataSet dataset: the training dataset :param int maxEpochs: max number of iterations to train the network :parma float learningrate: helps to :param float momentum: helps out of local minima while training, to get better results ''' self.trainer = BackpropTrainer(self.net, learningrate = learningrate, momentum = momentum) self.trainer.trainOnDataset(dataset, maxEpochs) def trainConvergence(self, dataset, maxEpochs = 10, learningrate = 0.01, momentum = 0.99, continueEpochs=10, validationProportion=0.25): '''trains a network with the given dataset nutil it converges :param SupervisedDataSet dataset: the training dataset :param int maxEpochs: max number of iterations to train the network :parma float learningrate: helps to :param float momentum: helps out of local minima while training, to get better results ''' self.trainer = BackpropTrainer(self.net, learningrate = learningrate, momentum = momentum) self.trainer.trainUntilConvergence(dataset, maxEpochs, False, continueEpochs, validationProportion) def test(self, data=None, verbose=False): if not self.trainer: raise ValueError("call train() first, to create a valid trainer object") return self.trainer.testOnData(data, verbose) def activate(self, value, rnd=False): inpt = self.net.activate(value)[0] if rnd: return self._clazz(inpt) return inpt def _clazz(self, inpt): clazz = round(inpt) if (clazz < 0): return 0 if (clazz > 1): return 1 return int(clazz) def save(self, name): '''saves the neural network :param string name: filename for the to be saved network ''' f = open(self.path + name + FILE_EXTENSION, 'w') pickle.dump(self.net, f) f.close() def load(self, name): '''loades the neural network :param string name: filename for the to be loaded network :return: instance of a saved neural network :rtype: NeuralNetwork ''' f = open(self.path + name + FILE_EXTENSION, 'r') self.net = pickle.load(f) f.close() return self def __repr__(self): return "%s\n%s" % (self.__class__.__name__, str(self.net))
__author__ = 'Stubborn' from pybrain.datasets.supervised import SupervisedDataSet from pybrain.tools.shortcuts import buildNetwork from pybrain.supervised.trainers.backprop import BackpropTrainer D = SupervisedDataSet(2, 1) # 2 imput --> 1 output D.addSample([0,0], [0]) D.addSample([0,1], [1]) D.addSample([1,0], [1]) D.addSample([1,1], [0]) # 4 kombinationer av input och dess output "OR" funktion N = buildNetwork(2, 4, 1) # multilayer perception? med 1 gomt lager T = BackpropTrainer(N, learningrate = 0.01, momentum = 0.99) # momentum = reduced learningrate print (('MSE before'), T.testOnData(D)) T.trainOnDataset(D, 1000) T.trainUntilConvergence() print (('MSE after'), T.testOnData(D)) print D
def trainm_NN(x1, x2, parametresNN, actualNet): start_time = time.time() # prepare data x1 = map(list, x1) x2 = map(list, x2) X = x1 + x2 print shape(X) y1 = ones((shape(x1)[0], 1)) # els positius son la classe '1' y2 = -1 * ones((shape(x2)[0], 1)) # els negatius son la classe '-1' Y = list(y1) + list(y2) Y = ravel(Y) #-----RED NUMERO 1 n1 = FeedForwardNetwork() inLayer = TanhLayer(199) hiddenLayer1 = TanhLayer(40) hiddenLayer2 = LinearLayer(3) hiddenLayer3 = TanhLayer(5) outLayer = TanhLayer(1) n1.addInputModule(inLayer) n1.addModule(hiddenLayer1) n1.addModule(hiddenLayer2) n1.addModule(hiddenLayer3) n1.addOutputModule(outLayer) in_to_hidden1 = FullConnection(inLayer, hiddenLayer1) hidden1_to_hidden2 = FullConnection(hiddenLayer1, hiddenLayer2) hidden2_to_hidden3 = FullConnection(hiddenLayer2, hiddenLayer3) hidden3_to_out = FullConnection(hiddenLayer3, outLayer) n1.addConnection(in_to_hidden1) n1.addConnection(hidden1_to_hidden2) n1.addConnection(hidden2_to_hidden3) n1.addConnection(hidden3_to_out) n1.sortModules() #Crida necesaria init de moduls interns #-----RED NUMERO 2 n2 = FeedForwardNetwork() inLayer = TanhLayer(199) hiddenLayer1 = TanhLayer(12) hiddenLayer2 = LinearLayer(3) outLayer = LinearLayer(1) n2.addInputModule(inLayer) n2.addModule(hiddenLayer1) n2.addModule(hiddenLayer2) n2.addOutputModule(outLayer) in_to_hidden1 = FullConnection(inLayer, hiddenLayer1) hidden1_to_hidden2 = FullConnection(hiddenLayer1, hiddenLayer2) hidden2_to_out = FullConnection(hiddenLayer2, outLayer) n2.addConnection(in_to_hidden1) n2.addConnection(hidden1_to_hidden2) n2.addConnection(hidden2_to_out) n2.sortModules() #Crida necesaria init de moduls interns #-----RED NUMERO 3 n3 = FeedForwardNetwork() inLayer = TanhLayer(199) hiddenLayer1 = TanhLayer(27) hiddenLayer2 = TanhLayer(5) outLayer = LinearLayer(1) n3.addInputModule(inLayer) n3.addModule(hiddenLayer1) n3.addModule(hiddenLayer2) n3.addOutputModule(outLayer) in_to_hidden1 = FullConnection(inLayer, hiddenLayer1) hidden1_to_hidden2 = FullConnection(hiddenLayer1, hiddenLayer2) hidden2_to_out = FullConnection(hiddenLayer2, outLayer) n3.addConnection(in_to_hidden1) n3.addConnection(hidden1_to_hidden2) n3.addConnection(hidden2_to_out) n3.sortModules() #Crida necesaria init de moduls interns #-----RED NUMERO 4 (implementar RELU) #Selection of the net that gonna be trained if (actualNet == 1): n = n1 elif (actualNet == 2): n = n2 elif (actualNet == 3): n = n3 else: #TODO n = n1 #Initialization weights if (actualNet == 1): r = math.sqrt(1 / ((199 + 70 + 3 + 5 + 1) * (1.0))) sizeOfNet = 199 * 70 + 70 * 3 + 3 * 5 + 5 elif (actualNet == 2): r = math.sqrt(1 / ((199 + 12 + 3 + 1) * (1.0))) sizeOfNet = 199 * 12 + 12 * 3 + 3 elif (actualNet == 3): r = math.sqrt(1 / ((199 + 27 + 5 + 1) * (1.0))) sizeOfNet = 199 * 27 + 27 * 5 + 5 else: #TODO r = math.sqrt(1 / ((199 + 40 + 3 + 5 + 1) * (1.0))) sizeOfNet = 199 * 40 + 40 * 3 + 3 * 5 + 5 weights_init = random.uniform(low=-r, high=r, size=(sizeOfNet, )) n._setParameters(weights_init) DS = ClassificationDataSet(199, nb_classes=1) for i in range(shape(X)[0]): DS.addSample(list(X[i]), Y[i]) #DS._convertToOneOfMany() # No -> volem nomes una sortida #DS.setField('class', DS.getField('target')) trainer = BackpropTrainer(n, dataset=DS, momentum=parametresNN['Momentum'], learningrate=parametresNN['learningrate'], verbose=parametresNN['verbose'], weightdecay=parametresNN['weightdecay'], batchlearning=parametresNN['batchlearning']) trainningErrors, validationErrors = trainer.trainUntilConvergence( maxEpochs=parametresNN['maxEpochs']) f = trainer.trainEpochs(parametresNN['maxEpochs']) print "Red Activa: ", actualNet, " Tiempo transcurrido: ", time.time( ) - start_time, " Error final training:", trainningErrors[ -1], " Error final validation:", validationErrors[-1] return n