def train(input, hidden, output, dataset):
"""
Method to build and train a neural network with backpropagation until it converges
:param input: input nodes for network (1)
:param hidden: hidden nodes for network
:param output: output nodes for network (1)
:param dataset: SupervisedDataSet, maps input->output: 1->1, 2->2 ... 8->8
:return: the trained neural network
"""
net = buildNetwork(input, hidden, output, hiddenclass=TanhLayer)
trainer = BackpropTrainer(net, dataset, learningrate=0.01)
trainer.trainUntilConvergence(verbose=False, validationProportion=0.15, maxEpochs=1000, continueEpochs=10)
return net
def get_trained_ann(dataset, ann=None, test_train_prop=0.25, max_epochs=50):
tstdata, trndata = dataset.splitWithProportion(test_train_prop)
trndata._convertToOneOfMany()
tstdata._convertToOneOfMany()
if not ann:
ann = build_ann(trndata.indim, trndata.outdim)
# ann = build_exp_ann(trndata.indim, trndata.outdim)
# trainer = RPropMinusTrainer(ann)
trainer = BackpropTrainer(ann, dataset=trndata,learningrate=0.01, momentum=0.5, verbose=True)
trnresult = tstresult = 0
# for i in range(10):
trainer.trainUntilConvergence(maxEpochs=max_epochs, verbose=True)
trnresult = percentError( trainer.testOnClassData(), trndata['class'] )
tstresult = percentError( trainer.testOnClassData(dataset=tstdata ), tstdata['class'] )
# print trnresult, tstresult
return ann, trnresult, tstresult
def NetworkTrain(trainDataSet, mnetwork=NetworkBuild(), file='NetworkDump.pkl',maxEpochs=100):
mnetwork = NetworkBuild(new = True)
assert len(mnetwork[0].inmodules) == len(mnetwork[1].keys())
print('DEBUG')
#print(trainDataSet)
print("lens " + str(len(trainDataSet[0][0])) + " " + str(len(mnetwork[0].inmodules)))
# 定义数据集的格式
DS = SupervisedDataSet(len(trainDataSet[0][0]), len(trainDataSet[0][1]))
for itrainDataSet in trainDataSet:
indata = itrainDataSet[0]
outdata = itrainDataSet[1]
DS.addSample(indata, outdata)
# 如果要获得里面的输入／输出时，可以用
# 如果要把数据集切分成训练集和测试集，可以用下面的语句，训练集：测试集＝8:2
# 为了方便之后的调用，可以把输入和输出拎出来
# 训练器采用BP算法
# verbose = True即训练时会把Total error打印出来，库里默认训练集和验证集的比例为4:1，可以在括号里更改
mnetwork[0].sortModules()
trainer = BackpropTrainer(mnetwork[0], DS, verbose=True, learningrate=0.01)
# 0.0575
# maxEpochs即你需要的最大收敛迭代次数，这里采用的方法是训练至收敛，我一般设为1000
trainer.trainUntilConvergence(maxEpochs=maxEpochs)
'''
for mod in mnetwork[0].modules:
print "Module:", mod.name
if mod.paramdim > 0:
print "--parameters:", mod.params
for conn in mnetwork[0].connections[mod]:
print "-connection to", conn.outmod.name
if conn.paramdim > 0:
print "- parameters", conn.params
if hasattr(mnetwork[0], "recurrentConns"):
print "Recurrent connections"
for conn in mnetwork[0].recurrentConns:
print "-", conn.inmod.name, " to", conn.outmod.name
if conn.paramdim > 0:
print "- parameters", conn.params
'''
pickle.dump(mnetwork, open(file, 'wb'))
return mnetwork
def train(self, x, y, class_number=-1):
self.__class_num = max(np.unique(y).size, class_number)
if max(y) == self.__class_num:
self.__class_zero_indexing = False
y = np.array([i - 1 for i in y])
DS = ClassificationDataSet(x.shape[1], nb_classes=self.__class_num)
DS.setField('input', x)
DS.setField('target', y.reshape(y.size, 1))
DS._convertToOneOfMany()
hidden_num = (DS.indim + DS.outdim) / 2
self.__pybrain_bpnn = buildNetwork(DS.indim, hidden_num, DS.outdim, bias=True, hiddenclass=SigmoidLayer, outclass=SoftmaxLayer)
trainer = BackpropTrainer(self.__pybrain_bpnn, dataset=DS, learningrate=0.07, lrdecay=1.0, momentum=0.6)
trainer.trainUntilConvergence(DS, maxEpochs=30)
def train(network_file, input_length, output_length, training_data_file,
learning_rate, momentum, stop_on_convergence, epochs, classify):
n = get_network(network_file)
if classify:
ds = ClassificationDataSet(int(input_length),
int(output_length) * 2)
ds._convertToOneOfMany()
else:
ds = SupervisedDataSet(int(input_length), int(output_length))
training_data = get_training_data(training_data_file)
NetworkManager.last_training_set_length = 0
for line in training_data:
data = [float(x) for x in line.strip().split(',') if x != '']
input_data = tuple(data[:(int(input_length))])
output_data = tuple(data[(int(input_length)):])
ds.addSample(input_data, output_data)
NetworkManager.last_training_set_length += 1
t = BackpropTrainer(n,
learningrate=learning_rate,
momentum=momentum,
verbose=True)
print "training network " + network_storage_path + network_file
if stop_on_convergence:
t.trainUntilConvergence(ds, epochs)
else:
if classify:
t.trainOnDataset(ds['class'], epochs)
else:
t.trainOnDataset(ds, epochs)
error = t.testOnData()
print "training done"
if not math.isnan(error):
save_network(n, network_file)
else:
print "error occured, network not saved"
print "network saved"
return error
def get_trained_ann(dataset, ann=None, test_train_prop=0.25, max_epochs=50):
tstdata, trndata = dataset.splitWithProportion(test_train_prop)
trndata._convertToOneOfMany()
tstdata._convertToOneOfMany()
if not ann:
ann = build_ann(trndata.indim, trndata.outdim)
# ann = build_exp_ann(trndata.indim, trndata.outdim)
# trainer = RPropMinusTrainer(ann)
trainer = BackpropTrainer(ann,
dataset=trndata,
learningrate=0.01,
momentum=0.5,
verbose=True)
trnresult = tstresult = 0
# for i in range(10):
trainer.trainUntilConvergence(maxEpochs=max_epochs, verbose=True)
trnresult = percentError(trainer.testOnClassData(), trndata['class'])
tstresult = percentError(trainer.testOnClassData(dataset=tstdata),
tstdata['class'])
# print trnresult, tstresult
return ann, trnresult, tstresult
def main():
# carregando o dataset do iris
dataset = montaDataset()
# carregando os datasets temporários
# separando 60% dos dados
# para treino e 30 % para teste
datasetTreinoTemporario, dadosRepartidosTemporario = dataset.splitWithProportion(
0.6)
# carregando os datasets temporários
# separando os dados repartidos
# em 50 % para teste e
# os outros 50 % para validação
datasetTesteTemporario, datasetValidacaoTemporario = dadosRepartidosTemporario.splitWithProportion(
0.5)
# montandos os datasets finais
datasetTreino = montaDatasetConvertido(datasetTreinoTemporario)
datasetTeste = montaDatasetConvertido(datasetTesteTemporario)
datasetValidacao = montaDatasetConvertido(datasetValidacaoTemporario)
# definindo a estrutura da rede adpatadas ao SoftmaxLayer
# com dimensão 4 de entrada
# com 20 neurônios na 1 camada intermediária
# com 2 camadas
# com dimensão 3 na saída
# terá como saída adaptada à classe SoftmaxLayer
network = buildNetwork(4, 20, 2, 3, outclass=SoftmaxLayer)
# convertendo os dados para o objeto ConvertToOneOfMany
datasetTreino._convertToOneOfMany()
datasetTeste._convertToOneOfMany()
datasetValidacao._convertToOneOfMany()
# criando a rede neural treinada
# passando a estrutura da rede neural
# dataset para treino
neuralNetwork = BackpropTrainer(network,
dataset=datasetTreino,
learningrate=0.02,
momentum=0.14,
verbose=True)
# treinando a rede até ela convergir
# e semparando os erros em um array de erro do treino
# e um array dos erros da validação
errosTreino, validacaoErros = neuralNetwork.trainUntilConvergence(
dataset=datasetTreino, maxEpochs=1000)
# criando a camada externa de teste
outTest = network.activateOnDataset(datasetTeste).argmax(axis=1)
print(5 * "-" + " Precisão de teste de treinamento " + 5 * "-" + "\n")
print("Precisão no teste : {} %".format(
100 - percentError(outTest, datasetTeste["class"])))
# criando a camada externa de validação
outVal = network.activateOnDataset(datasetValidacao).argmax(axis=1)
print("\n" + 5 * "-" + " Precisão de teste de treinamento " + 5 * "-" +
"\n")
print("Precisão na validação : {} %".format(
100 - percentError(outVal, datasetTeste["class"])))
# fazendo o plot gráfico dos erros
plt.plot(errosTreino, "b", validacaoErros, "r")
plt.show()
for conn in net.connections[mod]:
print >> net_output_file, "-connection to", conn.outmod.name
print "-connection to", conn.outmod.name
if conn.paramdim > 0:
print >> net_output_file, "- parameters", conn.params
print "- parameters", conn.params
if hasattr(net, "recurrentConns"):
print "Recurrent connections"
for conn in net.connections[mod]:
print >> net_output_file, "-", conn.inmod.name, " to", conn.outmod.name
print "-", conn.inmod.name, " to", conn.outmod.name
if conn.paramdim > 0:
print >> net_output_file, "- parameters", conn.params
print "- parameters", conn.params
trainer_message = trainer.trainUntilConvergence(maxEpochs=1000)
print >> net_output_file, "train-errors:"
net_output_file.write('[\t')
for i in range(len(trainer_message[0])):
net_output_file.write(str(trainer_message[0][i]))
if i != len(trainer_message[0]) - 1:
net_output_file.write('\t')
net_output_file.write(']\n')
print >> net_output_file, "valid-errors:"
net_output_file.write('[\t')
for i in range(len(trainer_message[1])):
net_output_file.write(str(trainer_message[1][i]))
if i != len(trainer_message[1]) - 1:
net_output_file.write('\t')
class NeuralNetwork(object):
"""
The neural network class does all the heavy lifting to incorporate pybrain
neural networks into the NowTrade ecosystem.
"""
def __init__(self,
train_data,
prediction_data,
network_type=FEED_FORWARD_NETWORK,
network_dataset_type=SUPERVISED_DATASET,
trainer_type=BACKPROP_TRAINER):
self.train_data = train_data
self.prediction_data = prediction_data
self.network_type = network_type
self.network_dataset_type = network_dataset_type
self.trainer_type = trainer_type
self.network = None
self.network_dataset = None
self.dataset = None
self.trainer = None
self.trained_iterations = 0
self.momentum = None
self.learning_rate = None
self.hidden_layers = None
self.prediction_window = None
self.logger = logger.Logger(self.__class__.__name__)
self.logger.info('train_data: %s prediction_data: %s, network_type: %s, \
network_dataset_type: %s, trainer_type: %s'
%(train_data, prediction_data, network_type, \
network_dataset_type, trainer_type))
def save(self):
"""
Returns the pickled trained/tested neural network as a string.
"""
return cPickle.dumps(self)
def save_to_file(self, filename):
"""
Saves a neural network to file for later use.
Look into pybrain.datasets.supervised.SupervisedDataSet.saveToFile()
http://pybrain.org/docs/api/datasets/superviseddataset.html
"""
file_handler = open(filename, 'wb')
cPickle.dump(self, file_handler)
file_handler.close()
def build_network(self, dataset, new=True, **kwargs):
"""
Builds a neural network using the dataset provided.
Expected keyword args:
- 'hidden_layers'
- 'prediction_window'
- 'learning_rate'
- 'momentum'
"""
self.hidden_layers = kwargs.get('hidden_layers', 3)
self.prediction_window = kwargs.get('prediction_window', 1)
self.learning_rate = kwargs.get('learning_rate', 0.1)
self.momentum = kwargs.get('momentum', 0.01)
if not new:
self.network.sorted = False
self.network.sortModules()
if self.network_dataset_type == SUPERVISED_DATASET:
self.ready_supervised_dataset(dataset)
else:
raise InvalidNetworkDatasetType()
else:
if self.network_type == FEED_FORWARD_NETWORK:
self.network = buildNetwork(len(self.train_data),
self.hidden_layers, 1)
else:
raise InvalidNetworkType()
if self.network_dataset_type == SUPERVISED_DATASET:
self.ready_supervised_dataset(dataset)
else:
raise InvalidNetworkDatasetType()
if self.trainer_type == BACKPROP_TRAINER:
self.trainer = BackpropTrainer(self.network,
learningrate=self.learning_rate,
momentum=self.momentum,
verbose=True)
self.trainer.setData(self.network_dataset)
else:
raise InvalidTrainerType()
def ready_supervised_dataset(self, dataset):
"""
Ready the supervised dataset for training.
@TODO: Need to randomize the data being fed to the network.
See randomBatches() here: http://pybrain.org/docs/api/datasets/superviseddataset.html
"""
self.network_dataset = SupervisedDataSet(len(self.train_data), 1)
# Currently only supports log function for normalizing data
training_values = np.log(dataset.data_frame[self.train_data])
results = np.log(dataset.data_frame[self.prediction_data].shift(
-self.prediction_window))
training_values['PREDICTION_%s' % self.prediction_data[0]] = results
training_values = training_values.dropna()
for _, row_data in enumerate(training_values.iterrows()):
_, data = row_data
sample = list(data[:-1])
result = [data[-1]]
self.network_dataset.addSample(sample, result)
def train(self, cycles=1):
"""
Trains the network the number of iteration specified in the cycles parameter.
"""
for _ in range(cycles):
res = self.trainer.train()
self.trained_iterations += 1
return res
def train_until_convergence(self,
max_cycles=1000,
continue_cycles=10,
validation_proportion=0.25):
"""
Wrapper around the pybrain BackpropTrainer trainUntilConvergence method.
@see: http://pybrain.org/docs/api/supervised/trainers.html
"""
self.trainer = \
self.trainer.trainUntilConvergence(maxEpochs=max_cycles,
continueEpochs=continue_cycles,
validationProportion=validation_proportion)
def _activate(self, data):
"""
Activates the network using the data specified.
Returns the network's prediction.
"""
return self.network.activate(data)[0]
def activate_all(self, data_frame):
"""
Activates the network for all values in the dataframe specified.
"""
dataframe = np.log(data_frame[self.train_data])
res = []
for _, row_data in enumerate(dataframe.iterrows()):
_, data = row_data
sample = list(data)
res.append(self._activate(sample))
return np.exp(res)
for conn in net.connections[mod]:
print >> net_output_file, "-connection to", conn.outmod.name
print "-connection to", conn.outmod.name
if conn.paramdim > 0:
print >> net_output_file, "- parameters", conn.params
print "- parameters", conn.params
if hasattr(net, "recurrentConns"):
print "Recurrent connections"
for conn in net.connections[mod]:
print >> net_output_file , "-", conn.inmod.name, " to", conn.outmod.name
print "-", conn.inmod.name, " to", conn.outmod.name
if conn.paramdim > 0:
print >> net_output_file, "- parameters", conn.params
print "- parameters", conn.params
trainer_message = trainer.trainUntilConvergence(maxEpochs=1000)
print >> net_output_file, "train-errors:"
net_output_file.write('[\t')
for i in range(len(trainer_message[0])):
net_output_file.write(str(trainer_message[0][i]))
if i!=len(trainer_message[0])-1:
net_output_file.write('\t')
net_output_file.write(']\n')
print >> net_output_file, "valid-errors:"
net_output_file.write('[\t')
for i in range(len(trainer_message[1])):
net_output_file.write(str(trainer_message[1][i]))
if i!=len(trainer_message[1])-1:
net_output_file.write('\t')
print 'build set'
alldata = ClassificationDataSet(dim, 1, nb_classes=2)
(data,label,items) = BinReader.readData(ur'F:\AliRecommendHomeworkData\1212新版\train15_17.expand.samp.norm.bin')
#(train,label,data) = BinReader.readData(r'C:\data\small\norm\train1217.bin')
for i in range(len(data)):
alldata.addSample(data[i],label[i])
tstdata, trndata = alldata.splitWithProportion(0.25)
trainer = BackpropTrainer(n,trndata,momentum=0.1,verbose=True,weightdecay=0.01)
print 'start'
#trainer.trainEpochs(1)
trainer.trainUntilConvergence(maxEpochs=2)
trnresult = percentError(trainer.testOnClassData(),trndata['class'])
tstresult = percentError(trainer.testOnClassData(dataset=tstdata), tstdata['class'])
print "epoch: %4d" % trainer.totalepochs, \
" train error: %5.2f%%" % trnresult, \
" test error: %5.2f%%" % tstresult
print 'get result'
#trainer.trainUntilConvergence()
#out = ClassificationDataSet(37,1)
#(test,label,items) = BinReader.readData(r'C:\data\homework\1218t5w.bin')
##(test,label,data) = BinReader.readData(r'C:\data\small\norm\test1218.bin')
#for i in range(len(test)):
trainingSet = SupervisedDataSet(len(championdictionary),1)
#Takes each game and turns it into a vector. -1 is stored if the champion is on the opposing team, 1 if the champion is on the player's team
#and 0 if it wasn't played. The vector is then fed into the Neural Network's Training Set
print "Adding Games to NN"
for game in aramdata.readlines():
aramgame = json.loads(game)
teammates,opponents, gameWin = DatabaseActions.GetResults(aramgame)
#writes a vector of which champions were on which team followed by the result
gamevector = [0]*len(championdictionary)
for champion in teammates:
gamevector[championdictionary[str(champion)]["Id"]-1] = 1
for champion in opponents:
gamevector[championdictionary[str(champion)]["Id"]-1] = -1
#Feeds that result into our Neural Network's training set
trainingSet.appendLinked(gamevector,int(gameWin))
#Creates a Backpropagation trainer and proceeds to train on our set. This step can take a while due to the volume of our data.
print "Training NN"
trainer = BackpropTrainer(predictionNet,trainingSet)
trainer.trainUntilConvergence(dataset = trainingSet, maxEpochs = 300, verbose = True, continueEpochs = 10, validationProportion=0.1)
#Saves our trained Network by exporting it to an XML file
NetworkWriter.writeToFile(predictionNet, "TrainedNetwork.xml")
aramdata.close()
#add entries to the dataset
for entry in result:
ds.addSample(entry[0], (entry[1]))
leftDs, rightDs = ds.splitWithProportion(training_set_percentage)
if (use_stored_weights == False):
print("successs rate over all names before training:")
print(validate_dataset(result, net))
print()
#train the network
print("starting to train the network")
trainer = BackpropTrainer(net, leftDs)
trainer.trainUntilConvergence(maxEpochs=max_epochs)
#store learned parameters to file
print("validate against the validation dataset")
print(validate_dataset(rightDs, net))
print()
with io.open('data/weights.json', 'w', encoding='utf8') as f:
f.write(json.dumps(net.params.tolist()))
else:
with io.open('data/weights.json') as f:
weights = f.read()
weights = json.loads(weights)
net._setParameters(weights)
def base_experiment():
(eval_dataset, eval_costset) = DomainFnApprox.make_evaluation_datasets()
random_train_dataset = SupervisedDataSet(2, 1)
random_train_costset = SupervisedDataSet(2, 1)
for i in range(RANDOM_TRAINING_SAMPLES):
x = random.uniform(X_MIN, X_MAX)
y = random.uniform(Y_MIN, Y_MAX)
z = FN(x, y)
z_cost = COST_FN(x, y)
random_train_dataset.addSample([x, y], [z])
random_train_costset.addSample([x, y], [z_cost])
value_network = buildNetwork(2,
80,
20,
1,
hiddenclass=SigmoidLayer,
bias=True)
value_trainer = BackpropTrainer(value_network,
learningrate=LEARNING_RATE,
momentum=MOMENTUM,
verbose=True)
print 'Value Network Topology:'
print value_network
cost_network = buildNetwork(2,
80,
20,
1,
hiddenclass=SigmoidLayer,
bias=True)
cost_trainer = BackpropTrainer(cost_network,
learningrate=LEARNING_RATE,
momentum=MOMENTUM,
verbose=True)
# test_derivatives(value_network, [1, 1])
# test_derivatives(cost_network, [1, 1])
print 'Value MSE before: %.4f' % value_trainer.testOnData(eval_dataset)
value_trainer.trainUntilConvergence(random_train_dataset,
continueEpochs=6,
maxEpochs=MAX_EPOCHS)
# value_trainer.trainOnDataset(random_train_dataset, 1000)
print 'Value MSE after: %.4f' % value_trainer.testOnData(eval_dataset)
print 'Cost MSE before: %.4f' % cost_trainer.testOnData(eval_costset)
cost_trainer.trainUntilConvergence(random_train_costset,
continueEpochs=6,
maxEpochs=MAX_EPOCHS)
# cost_trainer.trainOnDataset(random_train_costset, 1000)
print 'Cost MSE after: %.4f' % cost_trainer.testOnData(eval_costset)
# test_derivatives(value_network, [1, 1])
# test_derivatives(cost_network, [1, 1])
f_value = open('../data/learnedvalue.txt', 'w')
f_cost = open('../data/learnedcost.txt', 'w')
unit = (X_MAX - X_MIN) / (EVAL_SAMPLES_AXIS - 1)
for i in range(EVAL_SAMPLES_AXIS):
for j in range(EVAL_SAMPLES_AXIS):
x = X_MIN + i * unit
y = Y_MIN + j * unit
z = value_network.activate([x, y])
z_cost = cost_network.activate([x, y])
f_value.write('%f %f %f\n' % (x, y, z[0]))
f_cost.write('%f %f %f\n' % (x, y, z_cost[0]))
f_value.close()
f_cost.close()
hidden = int(argv[1]) # X
# Expects one dimensional input and target output
ds = SupervisedDataSet(1, 1)
for x in range(1, 9):
ds.addSample(x, x)
# 1 input, X hidden, 1 output
network = buildNetwork(1, hidden, 1, hiddenclass=TanhLayer)
# Init BackpropTrainer
trainer = BackpropTrainer(network, dataset=ds)
# Train until convergence
trainer.trainUntilConvergence(verbose=False,
validationData=0.15,
maxEpochs=1000,
continueEpochs=10)
# Activating the network on different integers such as the inputs in the data-set
print("// Hidden nodes: {}".format(hidden))
for x in range(1, 9):
print("{} --> {}".format(x, network.activate([x])[0]))
# Activating with decimal inputs outside of dataset
print("\nDecimal input outside dataset:")
for x in range(100, 109):
print("{} --> {}".format(x, network.activate([x * 0.1])[0]))
# Activating with negative numbers outside of data-set
print("\nNegative input:")
for x in range(1, 9):
class PyBrainNetwork(ANNWrapper):
def __init__(self, hidden_neurons=None, output_class="softmax",
learning_rate=0.01, lr_decay=1.0, momentum=0.0,
weight_decay=0.0, max_epochs=None, continue_epochs=10,
validation_percent=0.25, random_state=None):
super(PyBrainNetwork, self).__init__()
self.hidden_neurons = hidden_neurons
self.output_class = output_class
self.learning_rate = learning_rate
self.lr_decay = lr_decay
self.momentum = momentum
self.weight_decay = weight_decay
self.max_epochs = max_epochs
self.continue_epochs = continue_epochs
self.validation_percent = validation_percent
self.random_state = random_state
self.network_ = None
self.trainer_ = None
def fit(self, inp, y, sample_weight=None):
self.classes_, y = numpy.unique(y, return_inverse=True)
self.n_classes_ = len(self.classes_)
n_features = inp.shape[1]
random_state = check_random_state(self.random_state)
# We need to build an ImportanceDataSet from inp, y and sample_weight
dataset = ImportanceDataSet(n_features, self.n_classes_)
if sample_weight is None:
sample_weight = numpy.ones(len(y))
for x, label_pos, weight in izip(inp, y, sample_weight):
target = numpy.zeros(self.n_classes_)
target[label_pos] = 1
weight = weight * numpy.ones(self.n_classes_)
dataset.newSequence()
dataset.addSample(x, target, weight)
if self.hidden_neurons is None:
hidden_neurons = (n_features + self.n_classes_)/2
else:
hidden_neurons = self.hidden_neurons
self.network_ = buildNetwork(
n_features, hidden_neurons, self.n_classes_,
outclass=self._get_output_class()
)
# Set the initial parameters in a repeatable way
net_params = random_state.random_sample(self.network_.paramdim)
self.network_.params[:] = net_params
self.trainer_ = BackpropTrainer(
self.network_, dataset=dataset, learningrate=self.learning_rate,
lrdecay=self.lr_decay, momentum=self.momentum,
weightdecay=self.weight_decay
)
self.trainer_.trainUntilConvergence(
random_state, maxEpochs=self.max_epochs,
continueEpochs=self.continue_epochs,
validationProportion=self.validation_percent
)
return self
def activate(self, x):
return self.network_.activate(x)
def _get_output_class(self):
if self.output_class == "softmax":
return SoftmaxLayer
elif self.output_class == "tanh":
return TanhLayer
elif self.output_class == "sigmoid":
return SigmoidLayer
elif self.output_class == "linear":
return LinearLayer
else:
raise ValueError(
"output_class can be: softmax, tanh, sigmoid, linear"
)
class NeuralNetwork(object):
'''Neural network wrapper for the pybrain implementation
'''
def __init__(self):
self.path = os.path.dirname(
os.path.abspath(__file__)) + "/../../../data/"
self.net = None
self.trainer = None
def createNew(self, nInputs, nHiddenLayers, nOutput, bias):
'''builds a new neural network
:param int nInputs: the number of input nodes
:param int nHiddenLayers: the number of hidden layers
:parma int nOutputs: the number of output nodes
:param bool bias: if True an bias node will be added
:return: instance of a new neural network
:rtype: NeuralNetwork
'''
self.net = buildNetwork(nInputs,
nHiddenLayers,
nOutput,
bias=bias,
hiddenclass=TanhLayer)
return self
def train(self,
dataset,
maxEpochs=10,
learningrate=0.01,
momentum=0.99,
continueEpochs=10,
validationProportion=0.25):
'''trains a network with the given dataset
:param SupervisedDataSet dataset: the training dataset
:param int maxEpochs: max number of iterations to train the network
:parma float learningrate: helps to
:param float momentum: helps out of local minima while training, to get better results
'''
self.trainer = BackpropTrainer(self.net,
learningrate=learningrate,
momentum=momentum)
self.trainer.trainOnDataset(dataset, maxEpochs)
def trainConvergence(self,
dataset,
maxEpochs=10,
learningrate=0.01,
momentum=0.99,
continueEpochs=10,
validationProportion=0.25):
'''trains a network with the given dataset nutil it converges
:param SupervisedDataSet dataset: the training dataset
:param int maxEpochs: max number of iterations to train the network
:parma float learningrate: helps to
:param float momentum: helps out of local minima while training, to get better results
'''
self.trainer = BackpropTrainer(self.net,
learningrate=learningrate,
momentum=momentum)
self.trainer.trainUntilConvergence(dataset, maxEpochs, False,
continueEpochs,
validationProportion)
def test(self, data=None, verbose=False):
if not self.trainer:
raise ValueError(
"call train() first, to create a valid trainer object")
return self.trainer.testOnData(data, verbose)
def activate(self, value, rnd=False):
inpt = self.net.activate(value)[0]
if rnd:
return self._clazz(inpt)
return inpt
def _clazz(self, inpt):
clazz = round(inpt)
if (clazz < 0):
return 0
if (clazz > 1):
return 1
return int(clazz)
def _createFilePath(self, filePath, defaultPath=True):
if defaultPath:
return self.path + filePath + FILE_EXTENSION
return filePath + FILE_EXTENSION
def save(self, filePath, defaultPath=True):
'''saves the neural network
:param string filePath: filepath for the to be saved network
'''
with open(self._createFilePath(filePath, defaultPath), 'w') as f:
pickle.dump(self.net, f)
def load(self, filePath, defaultPath=True):
'''loades the neural network
:param string filePath: filepath for the to be loaded network
:return: instance of a saved neural network
:rtype: NeuralNetwork
'''
with open(self._createFilePath(filePath, defaultPath), 'r') as f:
self.net = pickle.load(f)
return self
def __repr__(self):
return "%s\n%s" % (self.__class__.__name__, str(self.net))
class NeuralNetwork(object):
"""
The neural network class does all the heavy lifting to incorporate pybrain
neural networks into the NowTrade ecosystem.
"""
def __init__(self, train_data, prediction_data, network_type=FEED_FORWARD_NETWORK,
network_dataset_type=SUPERVISED_DATASET,
trainer_type=BACKPROP_TRAINER):
self.train_data = train_data
self.prediction_data = prediction_data
self.network_type = network_type
self.network_dataset_type = network_dataset_type
self.trainer_type = trainer_type
self.network = None
self.network_dataset = None
self.dataset = None
self.trainer = None
self.trained_iterations = 0
self.momentum = None
self.learning_rate = None
self.hidden_layers = None
self.prediction_window = None
self.logger = logger.Logger(self.__class__.__name__)
self.logger.info('train_data: %s prediction_data: %s, network_type: %s, \
network_dataset_type: %s, trainer_type: %s'
%(train_data, prediction_data, network_type, \
network_dataset_type, trainer_type))
def save(self):
"""
Returns the pickled trained/tested neural network as a string.
"""
return cPickle.dumps(self)
def save_to_file(self, filename):
"""
Saves a neural network to file for later use.
Look into pybrain.datasets.supervised.SupervisedDataSet.saveToFile()
http://pybrain.org/docs/api/datasets/superviseddataset.html
"""
file_handler = open(filename, 'wb')
cPickle.dump(self, file_handler)
file_handler.close()
def build_network(self, dataset, new=True, **kwargs):
"""
Builds a neural network using the dataset provided.
Expected keyword args:
- 'hidden_layers'
- 'prediction_window'
- 'learning_rate'
- 'momentum'
"""
self.hidden_layers = kwargs.get('hidden_layers', 3)
self.prediction_window = kwargs.get('prediction_window', 1)
self.learning_rate = kwargs.get('learning_rate', 0.1)
self.momentum = kwargs.get('momentum', 0.01)
if not new:
self.network.sorted = False
self.network.sortModules()
if self.network_dataset_type == SUPERVISED_DATASET:
self.ready_supervised_dataset(dataset)
else: raise InvalidNetworkDatasetType()
else:
if self.network_type == FEED_FORWARD_NETWORK:
self.network = buildNetwork(len(self.train_data), self.hidden_layers, 1)
else: raise InvalidNetworkType()
if self.network_dataset_type == SUPERVISED_DATASET:
self.ready_supervised_dataset(dataset)
else: raise InvalidNetworkDatasetType()
if self.trainer_type == BACKPROP_TRAINER:
self.trainer = BackpropTrainer(self.network,
learningrate=self.learning_rate,
momentum=self.momentum,
verbose=True)
self.trainer.setData(self.network_dataset)
else: raise InvalidTrainerType()
def ready_supervised_dataset(self, dataset):
"""
Ready the supervised dataset for training.
@TODO: Need to randomize the data being fed to the network.
See randomBatches() here: http://pybrain.org/docs/api/datasets/superviseddataset.html
"""
self.network_dataset = SupervisedDataSet(len(self.train_data), 1)
# Currently only supports log function for normalizing data
training_values = np.log(dataset.data_frame[self.train_data])
results = np.log(dataset.data_frame[self.prediction_data].shift(-self.prediction_window))
training_values['PREDICTION_%s' %self.prediction_data[0]] = results
training_values = training_values.dropna()
for _, row_data in enumerate(training_values.iterrows()):
_, data = row_data
sample = list(data[:-1])
result = [data[-1]]
self.network_dataset.addSample(sample, result)
def train(self, cycles=1):
"""
Trains the network the number of iteration specified in the cycles parameter.
"""
for _ in range(cycles):
res = self.trainer.train()
self.trained_iterations += 1
return res
def train_until_convergence(self, max_cycles=1000, continue_cycles=10,
validation_proportion=0.25):
"""
Wrapper around the pybrain BackpropTrainer trainUntilConvergence method.
@see: http://pybrain.org/docs/api/supervised/trainers.html
"""
self.trainer = \
self.trainer.trainUntilConvergence(maxEpochs=max_cycles,
continueEpochs=continue_cycles,
validationProportion=validation_proportion)
def _activate(self, data):
"""
Activates the network using the data specified.
Returns the network's prediction.
"""
return self.network.activate(data)[0]
def activate_all(self, data_frame):
"""
Activates the network for all values in the dataframe specified.
"""
dataframe = np.log(data_frame[self.train_data])
res = []
for _, row_data in enumerate(dataframe.iterrows()):
_, data = row_data
sample = list(data)
res.append(self._activate(sample))
return np.exp(res)
return m, m_min, m_max
'''training dataset with random phase screens (2018.01.17)'''
mx = np.abs(trnslos_all).max()
ntrnslos_all = trnslos_all / mx
n_frame = 20
norm_inp = ntrnslos_all[:, :n_frame * 72]
trnds = SupervisedDataSet(72 * n_frame, 1)
#norm_inp, trnslos_min, trnslos_max = normalise(trnslos)
trnds.setField('input', norm_inp)
trn_tar = np.empty([6000, 1])
trn_tar[:, 0] = np.arange(5, 11).repeat(1000) / 15
trnds.setField('target', trn_tar)
'''learning process'''
net = buildNetwork(72 * n_frame, 1000, 1, hiddenclass=TanhLayer)
lr = 0.001
momentum = 0
lrdecay = 1
wdecay = 0
t = BackpropTrainer(net,
trnds,
learningrate=lr,
lrdecay=lrdecay,
momentum=momentum,
verbose=True,
batchlearning=False,
weightdecay=wdecay)
trnerr, vderr, trnData, vdData = t.trainUntilConvergence(
maxEpochs=30, validationProportion=0.1)
print value_network
cost_network = buildNetwork(1,
40,
20,
1,
hiddenclass=SigmoidLayer,
bias=True)
cost_trainer = BackpropTrainer(cost_network,
learningrate=0.01,
momentum=0.00,
verbose=True)
print 'Value MSE before: %.4f' % value_trainer.testOnData(eval_dataset)
value_trainer.trainUntilConvergence(dataset,
continueEpochs=6,
maxEpochs=500)
# value_trainer.trainOnDataset(dataset, 1000)
print 'Value MSE after: %.4f' % value_trainer.testOnData(eval_dataset)
print 'Cost MSE before: %.4f' % cost_trainer.testOnData(eval_costset)
cost_trainer.trainUntilConvergence(costset,
continueEpochs=6,
maxEpochs=500)
# cost_trainer.trainOnDataset(costset, 1000)
print 'Cost MSE after: %.4f' % cost_trainer.testOnData(eval_costset)
# print cost_network.params
f_value = open('../data2d/valueplot2ddata.txt', 'w')
f_cost = open('../data2d/costplot2ddata.txt', 'w')
plot_dataset = SupervisedDataSet(2, 1)
class NeuralNetwork(object):
'''Neural network wrapper for the pybrain implementation
'''
def __init__(self):
self.path = os.path.dirname(os.path.abspath(__file__)) + "/../../data/"
self.net = None
self.trainer = None
def createNew(self, nInputs, nHiddenLayers, nOutput, bias):
'''builds a new neural network
:param int nInputs: the number of input nodes
:param int nHiddenLayers: the number of hidden layers
:parma int nOutputs: the number of output nodes
:param bool bias: if True an bias node will be added
:return: instance of a new neural network
:rtype: NeuralNetwork
'''
self.net = buildNetwork(nInputs, nHiddenLayers, nOutput, bias=bias, hiddenclass=TanhLayer)
return self
def train(self, dataset, maxEpochs = 10, learningrate = 0.01, momentum = 0.99, continueEpochs=10, validationProportion=0.25):
'''trains a network with the given dataset
:param SupervisedDataSet dataset: the training dataset
:param int maxEpochs: max number of iterations to train the network
:parma float learningrate: helps to
:param float momentum: helps out of local minima while training, to get better results
'''
self.trainer = BackpropTrainer(self.net, learningrate = learningrate, momentum = momentum)
self.trainer.trainOnDataset(dataset, maxEpochs)
def trainConvergence(self, dataset, maxEpochs = 10, learningrate = 0.01, momentum = 0.99, continueEpochs=10, validationProportion=0.25):
'''trains a network with the given dataset nutil it converges
:param SupervisedDataSet dataset: the training dataset
:param int maxEpochs: max number of iterations to train the network
:parma float learningrate: helps to
:param float momentum: helps out of local minima while training, to get better results
'''
self.trainer = BackpropTrainer(self.net, learningrate = learningrate, momentum = momentum)
self.trainer.trainUntilConvergence(dataset, maxEpochs, False, continueEpochs, validationProportion)
def test(self, data=None, verbose=False):
if not self.trainer:
raise ValueError("call train() first, to create a valid trainer object")
return self.trainer.testOnData(data, verbose)
def activate(self, value, rnd=False):
inpt = self.net.activate(value)[0]
if rnd:
return self._clazz(inpt)
return inpt
def _clazz(self, inpt):
clazz = round(inpt)
if (clazz < 0):
return 0
if (clazz > 1):
return 1
return int(clazz)
def save(self, name):
'''saves the neural network
:param string name: filename for the to be saved network
'''
f = open(self.path + name + FILE_EXTENSION, 'w')
pickle.dump(self.net, f)
f.close()
def load(self, name):
'''loades the neural network
:param string name: filename for the to be loaded network
:return: instance of a saved neural network
:rtype: NeuralNetwork
'''
f = open(self.path + name + FILE_EXTENSION, 'r')
self.net = pickle.load(f)
f.close()
return self
def __repr__(self):
return "%s\n%s" % (self.__class__.__name__, str(self.net))
__author__ = 'Stubborn'
from pybrain.datasets.supervised import SupervisedDataSet
from pybrain.tools.shortcuts import buildNetwork
from pybrain.supervised.trainers.backprop import BackpropTrainer
D = SupervisedDataSet(2, 1)
# 2 imput --> 1 output
D.addSample([0,0], [0])
D.addSample([0,1], [1])
D.addSample([1,0], [1])
D.addSample([1,1], [0])
# 4 kombinationer av input och dess output "OR" funktion
N = buildNetwork(2, 4, 1)
# multilayer perception? med 1 gomt lager
T = BackpropTrainer(N, learningrate = 0.01, momentum = 0.99)
# momentum = reduced learningrate
print (('MSE before'), T.testOnData(D))
T.trainOnDataset(D, 1000)
T.trainUntilConvergence()
print (('MSE after'), T.testOnData(D))
print D
def trainm_NN(x1, x2, parametresNN, actualNet):
start_time = time.time()
# prepare data
x1 = map(list, x1)
x2 = map(list, x2)
X = x1 + x2
print shape(X)
y1 = ones((shape(x1)[0], 1)) # els positius son la classe '1'
y2 = -1 * ones((shape(x2)[0], 1)) # els negatius son la classe '-1'
Y = list(y1) + list(y2)
Y = ravel(Y)
#-----RED NUMERO 1
n1 = FeedForwardNetwork()
inLayer = TanhLayer(199)
hiddenLayer1 = TanhLayer(40)
hiddenLayer2 = LinearLayer(3)
hiddenLayer3 = TanhLayer(5)
outLayer = TanhLayer(1)
n1.addInputModule(inLayer)
n1.addModule(hiddenLayer1)
n1.addModule(hiddenLayer2)
n1.addModule(hiddenLayer3)
n1.addOutputModule(outLayer)
in_to_hidden1 = FullConnection(inLayer, hiddenLayer1)
hidden1_to_hidden2 = FullConnection(hiddenLayer1, hiddenLayer2)
hidden2_to_hidden3 = FullConnection(hiddenLayer2, hiddenLayer3)
hidden3_to_out = FullConnection(hiddenLayer3, outLayer)
n1.addConnection(in_to_hidden1)
n1.addConnection(hidden1_to_hidden2)
n1.addConnection(hidden2_to_hidden3)
n1.addConnection(hidden3_to_out)
n1.sortModules() #Crida necesaria init de moduls interns
#-----RED NUMERO 2
n2 = FeedForwardNetwork()
inLayer = TanhLayer(199)
hiddenLayer1 = TanhLayer(12)
hiddenLayer2 = LinearLayer(3)
outLayer = LinearLayer(1)
n2.addInputModule(inLayer)
n2.addModule(hiddenLayer1)
n2.addModule(hiddenLayer2)
n2.addOutputModule(outLayer)
in_to_hidden1 = FullConnection(inLayer, hiddenLayer1)
hidden1_to_hidden2 = FullConnection(hiddenLayer1, hiddenLayer2)
hidden2_to_out = FullConnection(hiddenLayer2, outLayer)
n2.addConnection(in_to_hidden1)
n2.addConnection(hidden1_to_hidden2)
n2.addConnection(hidden2_to_out)
n2.sortModules() #Crida necesaria init de moduls interns
#-----RED NUMERO 3
n3 = FeedForwardNetwork()
inLayer = TanhLayer(199)
hiddenLayer1 = TanhLayer(27)
hiddenLayer2 = TanhLayer(5)
outLayer = LinearLayer(1)
n3.addInputModule(inLayer)
n3.addModule(hiddenLayer1)
n3.addModule(hiddenLayer2)
n3.addOutputModule(outLayer)
in_to_hidden1 = FullConnection(inLayer, hiddenLayer1)
hidden1_to_hidden2 = FullConnection(hiddenLayer1, hiddenLayer2)
hidden2_to_out = FullConnection(hiddenLayer2, outLayer)
n3.addConnection(in_to_hidden1)
n3.addConnection(hidden1_to_hidden2)
n3.addConnection(hidden2_to_out)
n3.sortModules() #Crida necesaria init de moduls interns
#-----RED NUMERO 4 (implementar RELU)
#Selection of the net that gonna be trained
if (actualNet == 1):
n = n1
elif (actualNet == 2):
n = n2
elif (actualNet == 3):
n = n3
else:
#TODO
n = n1
#Initialization weights
if (actualNet == 1):
r = math.sqrt(1 / ((199 + 70 + 3 + 5 + 1) * (1.0)))
sizeOfNet = 199 * 70 + 70 * 3 + 3 * 5 + 5
elif (actualNet == 2):
r = math.sqrt(1 / ((199 + 12 + 3 + 1) * (1.0)))
sizeOfNet = 199 * 12 + 12 * 3 + 3
elif (actualNet == 3):
r = math.sqrt(1 / ((199 + 27 + 5 + 1) * (1.0)))
sizeOfNet = 199 * 27 + 27 * 5 + 5
else:
#TODO
r = math.sqrt(1 / ((199 + 40 + 3 + 5 + 1) * (1.0)))
sizeOfNet = 199 * 40 + 40 * 3 + 3 * 5 + 5
weights_init = random.uniform(low=-r, high=r, size=(sizeOfNet, ))
n._setParameters(weights_init)
DS = ClassificationDataSet(199, nb_classes=1)
for i in range(shape(X)[0]):
DS.addSample(list(X[i]), Y[i])
#DS._convertToOneOfMany() # No -> volem nomes una sortida
#DS.setField('class', DS.getField('target'))
trainer = BackpropTrainer(n,
dataset=DS,
momentum=parametresNN['Momentum'],
learningrate=parametresNN['learningrate'],
verbose=parametresNN['verbose'],
weightdecay=parametresNN['weightdecay'],
batchlearning=parametresNN['batchlearning'])
trainningErrors, validationErrors = trainer.trainUntilConvergence(
maxEpochs=parametresNN['maxEpochs'])
f = trainer.trainEpochs(parametresNN['maxEpochs'])
print "Red Activa: ", actualNet, " Tiempo transcurrido: ", time.time(
) - start_time, " Error final training:", trainningErrors[
-1], " Error final validation:", validationErrors[-1]
return n
