def createAndTrainNetworkFromFile(curs_filename,
count_input_samples,
count_samples,
net_filename,
count_layers=33,
count_outputs=1,
max_epochs=15000,
min_epochs=300):
net = buildNetwork(count_input_samples, count_layers, count_outputs)
ds = SupervisedDataSet(count_input_samples, count_outputs)
wb = load_workbook(filename=curs_filename)
ws = wb.active
for i in range(0, count_samples):
loaded_data = []
for j in range(0, count_input_samples + 1):
loaded_data.append(
round(float(ws.cell(row=i + 1, column=j + 1).value), 4))
#ds.addSample(loaded_data[:-1], loaded_data[-1])
#print loaded_data[:-1], loaded_data[-1]
ds.addSample(loaded_data[:-1], loaded_data[-1])
trainer = RPropMinusTrainer(net, verbose=True)
trainer.setData(ds)
a = trainer.trainUntilConvergence(maxEpochs=max_epochs,
continueEpochs=min_epochs,
validationProportion=0.15)
net_filename = net_filename[:-4] + str(a[0][-1]) + '.xml'
NetworkWriter.writeToFile(net, net_filename)
result_list = [a, net_filename]
return result_list
def train(
train,
label,
custom_net=None,
training_mse_threshold=0.40,
testing_mse_threshold=0.60,
epoch_threshold=10,
epochs=100,
hidden_size=20,
):
# Test Set.
x_train = train[0:split_at, :]
y_train_slice = label.__getslice__(0, split_at)
y_train = y_train_slice.reshape(-1, 1)
x_test = train[split_at:, :]
y_test_slice = label.__getslice__(split_at, label.shape[0])
y_test = y_test_slice.reshape(-1, 1)
# Shape.
input_size = x_train.shape[1]
target_size = y_train.shape[1]
# prepare dataset
ds = SDS(input_size, target_size)
ds.setField("input", x_train)
ds.setField("target", y_train)
# prepare dataset
ds_test = SDS(input_size, target_size)
ds_test.setField("input", x_test)
ds_test.setField("target", y_test)
min_mse = 1000000
# init and train
if custom_net == None:
net = buildNetwork(input_size, hidden_size, target_size, bias=True)
else:
print "Picking up the custom network"
net = custom_net
trainer = RPropMinusTrainer(net, dataset=ds, verbose=False, weightdecay=0.01, batchlearning=True)
print "training for {} epochs...".format(epochs)
for i in range(epochs):
mse = trainer.train()
print "training mse, epoch {}: {}".format(i + 1, math.sqrt(mse))
p = net.activateOnDataset(ds_test)
mse = math.sqrt(MSE(y_test, p))
print "-- testing mse, epoch {}: {}".format(i + 1, mse)
pickle.dump(net, open("current_run", "wb"))
if min_mse > mse:
print "Current minimum found at ", i
pickle.dump(net, open("current_min_epoch_" + model_file, "wb"))
min_mse = mse
pickle.dump(net, open(model_file, "wb"))
return net
def exam(self, dc, train_com, train_path):
"""
Here you can train your networks.
Parameters
----------
:param dc: dict
Dict of commands with values.
:param train_com:
Command what you want teach by ann to recognize.
:param train_path:
Path to folder with train examples.
Returns
-------
:return:
File with network.
"""
num_hid = 1
put, out = [], []
ds = SupervisedDataSet(420, 1)
nt = buildNetwork(420, 3, 1, bias=True, hiddenclass=SigmoidLayer, outclass=SigmoidLayer)
for way in train_path:
for i in os.listdir(way):
lk = self.link(i)
if lk:
self.logger.debug(u'File was added to training list %s' % i)
result = self.ext_t(way+i)
ds.addSample(result, (dc[lk],))
put.append(result)
out.append([dc[lk]])
net = nl.net.newff([[np.min(put), np.max(put)]]*420, [num_hid, 1], [nl.trans.LogSig(), nl.trans.SatLinPrm()])
net.trainf = nl.train.train_rprop
trainer = RPropMinusTrainer(nt, dataset=ds, verbose=False)
self.logger.info(u'Training brain...')
trainer.trainUntilConvergence(maxEpochs=100, verbose=False, continueEpochs=100, validationProportion=1e-7)
self.logger.info(u'Training neural...')
error = net.train(put, out, epochs=500, show=500, goal=1e-4, lr=1e-10)
while error[-1] > 1e-3:
self.logger.info(u'Try to one more training, because MSE are little not enough!')
net = nl.net.newff([[np.min(put), np.max(put)]]*420, [num_hid, 1], [nl.trans.LogSig(), nl.trans.SatLinPrm()])
net.trainf = nl.train.train_rprop
self.logger.info(u'Training neural...')
error = net.train(put, out, epochs=500, show=500, goal=1e-4, lr=1e-10)
num_hid += 1
try:
net.save(u'networks/%s_neurolab' % train_com)
fl = open(u'networks/%s_brain' % train_com, 'w')
pickle.dump(nt, fl)
fl.close()
except IOError:
os.mkdir(u'networks')
net.save(u'networks/%s_neurolab' % train_com)
def train(self, epoch):
self.ds._convertToOneOfMany()
trainer = RPropMinusTrainer(self.net,
dataset=self.ds,
momentum=0.1,
verbose=True,
weightdecay=0.01)
trainer.trainEpochs(epoch)
def trainNetwork(net, sample_list, validate_list, net_filename, max_epochs=5500, min_epochs=300):
count_input_samples = len(sample_list)
count_outputs = len(validate_list)
ds = SupervisedDataSet(count_input_samples, count_outputs)
ds.addSample(sample_list, validate_list)
trainer = RPropMinusTrainer(net, verbose=True)
trainer.setData(ds)
trainer.trainUntilConvergence(maxEpochs=max_epochs, continueEpochs=min_epochs)
NetworkWriter.writeToFile(net, net_filename)
return net
def train_cross_validate(train, label, custom_net=None, training_mse_threshold=0.40, testing_mse_threshold=0.60,
epoch_threshold=10, epochs=100, hidden_size=50):
# Test Set.
x_train = train[0:split_at, :]
y_train_slice = label.__getslice__(0, split_at)
y_train = y_train_slice.reshape(-1, 1)
x_test = train[split_at:, :]
y_test_slice = label.__getslice__(split_at, label.shape[0])
y_test = y_test_slice.reshape(-1, 1)
# Shape.
input_size = x_train.shape[1]
target_size = y_train.shape[1]
input_size_test = x_test.shape[1]
target_size_test = y_test.shape[1]
# prepare dataset
ds = SDS(input_size, target_size)
ds.setField('input', x_train)
ds.setField('target', y_train)
# prepare dataset
ds_test = SDS(input_size, target_size)
ds_test.setField('input', x_test)
ds_test.setField('target', y_test)
min_mse = 1000000
# init and train
if custom_net == None:
net = buildNetwork(input_size, hidden_size, target_size, bias=True, hiddenclass=TanhLayer)
else:
print "Picking up the custom network"
net = custom_net
trainer = RPropMinusTrainer(net, dataset=ds, verbose=True, weightdecay=0.01, batchlearning=True)
print "training for {} epochs...".format(epochs)
for i in range(epochs):
mse = trainer.train()
print "training mse, epoch {}: {}".format(i + 1, mse)
p = net.activateOnDataset(ds_test)
mse = MSE(y_test, p)
print "-- testing mse, epoch {}: {}".format(i + 1, mse)
pickle.dump(net, open("current_run", 'wb'))
if min_mse > mse:
print "Current minimum found at ", i
pickle.dump(net, open("current_min_epoch_" + model_file, 'wb'))
min_mse = mse
pickle.dump(net, open(model_file, 'wb'))
return net
def recurrent_neural_network(self, train_X, train_y, test_X, test_y, n_hidden_neurons=50, iterations=100,
gridsearch=False, gridsearch_training_frac=0.7, outputbias=False, error='accuracy'):
"""
Apply a recurrent neural network for classification upon the training data (with the specified number of
hidden neurons and iterations), and use the created network to predict the outcome for both the test and
training set. It returns the categorical predictions for the training and test set as well as the
probabilities associated with each class, each class being represented as a column in the data frame.
"""
if gridsearch:
n_hidden_neurons, iterations, outputbias = self. \
gridsearch_recurrent_neural_network(train_X, train_y, gridsearch_training_frac=gridsearch_training_frac,
error=error)
# Create numerical datasets first
new_train_X, new_test_X = self.create_numerical_multiple_dataset(train_X, test_X)
new_train_y, new_test_y = self.create_numerical_multiple_dataset(train_y, test_y)
# Normalize the input
new_train_X, new_test_X, min_X, max_X = self.normalize(new_train_X, new_test_X, 0, 1)
new_train_y, new_test_y, min_y, max_y = self.normalize(new_train_y, new_test_y, 0.1, 0.9)
# Create the proper pybrain datasets
ds_training = self.rnn_dataset(new_train_X, new_train_y)
ds_test = self.rnn_dataset(new_test_X, new_test_y)
inputs = len(new_train_X.columns)
outputs = len(new_train_y.columns)
# Build the network with the proper parameters
n = buildNetwork(inputs, n_hidden_neurons, outputs, hiddenclass=SigmoidLayer, outclass=SigmoidLayer,
outputbias=outputbias, recurrent=True)
# Train using back propagation through time
# trainer = BackpropTrainer(n, dataset=ds_training, verbose=False, momentum=0.9, learningrate=0.01)
trainer = RPropMinusTrainer(n, dataset=ds_training, verbose=False)
for i in range(0, iterations):
trainer.train()
Y_train = []
Y_test = []
for sample, target in ds_training.getSequenceIterator(0):
Y_train.append(n.activate(sample).tolist())
for sample, target in ds_test.getSequenceIterator(0):
Y_test.append(n.activate(sample).tolist())
y_train_result = pd.DataFrame(Y_train, columns=new_train_y.columns, index=train_y.index)
y_test_result = pd.DataFrame(Y_test, columns=new_test_y.columns, index=test_y.index)
y_train_result = self.denormalize(y_train_result, min_y, max_y, 0.1, 0.9)
y_test_result = self.denormalize(y_test_result, min_y, max_y, 0.1, 0.9)
return y_train_result.idxmax(axis=1), y_test_result.idxmax(axis=1), y_train_result, y_test_result
def train(self, params, verbose=False):
if params['reset_every_training']:
if verbose:
print 'create lstm network'
random.seed(6)
if params['output_encoding'] == None:
self.net = buildNetwork(self.nDimInput,
params['num_cells'],
self.nDimOutput,
hiddenclass=LSTMLayer,
bias=True,
outputbias=True,
recurrent=True)
elif params['output_encoding'] == 'likelihood':
self.net = buildNetwork(self.nDimInput,
params['num_cells'],
self.nDimOutput,
hiddenclass=LSTMLayer,
bias=True,
outclass=SigmoidLayer,
recurrent=True)
self.net.reset()
ds = SequentialDataSet(self.nDimInput, self.nDimOutput)
networkInput = self.window(self.networkInput, params)
targetPrediction = self.window(self.targetPrediction, params)
# prepare a training data-set using the history
for i in xrange(len(networkInput)):
ds.addSample(self.inputEncoder.encode(networkInput[i]),
self.outputEncoder.encode(targetPrediction[i]))
mycount = 0
if params['num_epochs'] > 1:
trainer = RPropMinusTrainer(self.net, dataset=ds, verbose=verbose)
if verbose:
print " train LSTM on ", len(
ds), " records for ", params['num_epochs'], " epochs "
if len(networkInput) > 1:
trainer.trainEpochs(params['num_epochs'])
else:
self.trainer.setData(ds)
self.trainer.train()
# run through the training dataset to get the lstm network state right
self.net.reset()
for i in xrange(len(networkInput)):
self.net.activate(ds.getSample(i)[0])
def AddData(self, datainput, dataoutput):
if len(dataoutput) != len(datainput):
print("Not equals data", len(dataoutput), len(datainput))
return 1
self.ds = SupervisedDataSet(self.inputsize, self.outputsize)
for i in xrange(len(dataoutput)):
self.ds.appendLinked(datainput[i], dataoutput[i])
self.trainer = RPropMinusTrainer(self.net,
dataset=self.ds,
learningrate=0.1)
return 0
def createAndTrainNetworkFromList(train_list, count_input_samples, net_filename, count_layers=33,
count_outputs=1, max_epochs=15000, min_epochs=300):
net = buildNetwork(count_input_samples, count_layers, count_outputs)
ds = SupervisedDataSet(count_input_samples, count_outputs)
count_samples = len(train_list)
for i in range(0, count_samples):
ds.addSample(train_list[i][:-count_outputs], train_list[i][-count_outputs])
trainer = RPropMinusTrainer(net, verbose=True)
trainer.setData(ds)
a = trainer.trainUntilConvergence(maxEpochs=max_epochs, continueEpochs=min_epochs, validationProportion=0.15)
net_filename = net_filename[:-4]+str(a[0][-1])+'.xml'
NetworkWriter.writeToFile(net, net_filename)
result_list = [a, net_filename]
return result_list
def __init__(self, ds):
num_inputs = ds.num_features * sequence_classifier.num_time_steps
self.alldata = SequenceClassificationDataSet(num_inputs,
target = 1,
nb_classes = ds.get_num_classes(),
class_labels = ds.get_classes() )
for Idx in range(len(ds.all_moves)):
if not (Idx + sequence_classifier.num_time_steps < len(ds.all_moves)):
continue
class_first = ds.all_moves[Idx].class_
features = []
for i in range(sequence_classifier.num_time_steps):
features = features + ds.all_moves[Idx + i].get_features()
class_last = ds.all_moves[Idx + sequence_classifier.num_time_steps].class_
if class_first == class_last:
self.alldata.appendLinked(features, [ds.get_classes().index(ds.all_moves[Idx].class_)])
self.alldata.newSequence()
self.tstdata, self.trndata = self.alldata.splitWithProportion(0.25)
self.trndata._convertToOneOfMany()
self.tstdata._convertToOneOfMany()
self.seq_rnn = None #buildNetwork(num_inputs, 2, self.trndata.outdim, hiddenclass=LSTMLayer,recurrent=True ,outclass=SoftmaxLayer)
self.create_network(num_inputs)
self.trainer = RPropMinusTrainer(module=self.seq_rnn, dataset=self.trndata)
def train(self, training_files, learningrate=0.01, scaling=True, noise=False, verbose=True):
print "building dataset..."
ds = SupervisedDataSet(SensorModel.array_length(self.sensor_ids), 1)
# read training file line, create sensormodel object, do backprop
a = None
s = None
for logfile in training_files:
print "loading file", logfile
with open(logfile) as f:
for line in f:
if line.startswith("Received:"):
s = SensorModel(string=line.split(' ', 1)[1])
elif line.startswith("Sending:"):
a = Actions.from_string(string=line.split(' ', 1)[1])
if s is not None and a is not None:
ds.addSample(inp=s.get_array(self.sensor_ids), target=a[self.action_ids[0]])
if noise:
# add the same training sample again but with noise in the sensors
s.add_noise()
ds.addSample(inp=s.get_array(self.sensor_ids), target=a[self.action_ids[0]])
s = None
a = None
print "dataset size:", len(ds)
if scaling:
print "scaling dataset"
self.scaler_input = StandardScaler(with_mean=True, with_std=False).fit(ds.data['input'])
ds.data['input'] = self.scaler_input.transform(ds.data['input'])
ds.data['target'] = ds.data['target']
#self.trainer = BackpropTrainer(self.net, learningrate=learningrate, verbose=verbose)
self.trainer = RPropMinusTrainer(self.net, verbose=verbose, batchlearning=True)
print "training network..."
self.trainer.trainUntilConvergence(dataset=ds, validationProportion=0.25, maxEpochs=10, continueEpochs=2)
def __init__(self):
self.inputs = 3
self.outputs = 1
self.n = buildNetwork(self.inputs, 200,200,200,200,self.outputs, bias=True,hiddenclass=TanhLayer)
self.n.sortModules()
self.ds = SupervisedDataSet(self.inputs, self.outputs)
self.trainer = RPropMinusTrainer(self.n)
self.trainer.setData(self.ds)
def train(self, trndata, valdata, hidden_neurons=5, hidden_class=SigmoidLayer, iterations=3):
print "Hidden neurons: " + str(hidden_neurons)
print "Hidden class: " + str(hidden_class)
print "Iterations: " + str(iterations)
fnn = buildNetwork(trndata.indim, hidden_neurons, trndata.outdim, outclass=SoftmaxLayer,
hiddenclass=hidden_class)
trainer = RPropMinusTrainer(fnn, dataset=trndata, verbose=False)
#trainer = BackpropTrainer(fnn, dataset=trndata, momentum=0.5, verbose=True, learningrate=0.05)
for i in range(iterations):
trainer.train()
out, tar = trainer.testOnClassData(dataset=valdata, return_targets=True, verbose=False)
#used to return final score, not used yet :D
print str(i) + " " + str(accuracy(out, tar))
self.model = trainer
def AddDataSequential(self, data):
self.ds = SequentialDataSet(self.inputsize, self.outputsize)
for i in xrange(len(data) - 1, 0, -1):
t = data[i]
k = i - 1
while k > -1:
self.ds.appendLinked(data[k], t)
k -= 1
self.ds.newSequence()
"""print self.ds.getNumSequences()
for i in range(self.ds.getNumSequences()):
for input, target in self.ds.getSequenceIterator(i):
print i, TransToIntList_45(input), TransToIntList_45(target)"""
self.trainer = RPropMinusTrainer(self.net,
dataset=self.ds,
learningrate=0.1)
return 0
def train_net():
fnn = buildNetwork(len(input_args), 3, 2)
ds = ClassificationDataSet(len(input_args),2,nb_classes=2)
ds = generate_data(ds , hour_to_use_app = 10)
trainer = RPropMinusTrainer( fnn, dataset= ds, verbose=True)
trainer.train()
trainer.trainEpochs(15)
test = ClassificationDataSet(4,2)
test.addSample((12,6,10,6),[1,0])
test.addSample((12,1,7,2),[0,1])
test.addSample((12,3,11,1),[0,1])
fnn.activateOnDataset(test)
return fnn,trainer,ds,test
def createAndTrainNetworkFromFile(curs_filename, count_input_samples, count_samples, net_filename, count_layers=33,
count_outputs=1, max_epochs=15000, min_epochs=300):
net = buildNetwork(count_input_samples, count_layers, count_outputs)
ds = SupervisedDataSet(count_input_samples, count_outputs)
wb = load_workbook(filename=curs_filename)
ws = wb.active
for i in range(0, count_samples):
loaded_data = []
for j in range(0, count_input_samples + 1):
loaded_data.append(round(float(ws.cell(row=i+1, column=j+1).value), 4))
#ds.addSample(loaded_data[:-1], loaded_data[-1])
#print loaded_data[:-1], loaded_data[-1]
ds.addSample(loaded_data[:-1], loaded_data[-1])
trainer = RPropMinusTrainer(net, verbose=True)
trainer.setData(ds)
a = trainer.trainUntilConvergence(maxEpochs=max_epochs, continueEpochs=min_epochs, validationProportion=0.15)
net_filename = net_filename[:-4]+str(a[0][-1])+'.xml'
NetworkWriter.writeToFile(net, net_filename)
result_list = [a, net_filename]
return result_list
def train(self,
cycles,
percent,
hidden_layers=3,
hiddenclass=None,
num_outputs=1,
num_inputs=-1):
num_inputs = self._count_inputs() if num_inputs == -1 else num_inputs
if num_inputs <= 0 or num_outputs <= 0 or cycles <= 0 or (percent > 100 or percent <= 0):
return
network = self._buildNet(hidden_layers, num_outputs, num_inputs, hiddenclass)
data_set = self.get_data_set(percent, num_inputs, num_outputs)
trainer = RPropMinusTrainer(network, dataset=data_set)
for i in range(cycles):
trainer.train()
return network
class Brain():
def __init__(self):
self.inputs = 3
self.outputs = 1
self.n = buildNetwork(self.inputs, 200,200,200,200,self.outputs, bias=True,hiddenclass=TanhLayer)
self.n.sortModules()
self.ds = SupervisedDataSet(self.inputs, self.outputs)
self.trainer = RPropMinusTrainer(self.n)
self.trainer.setData(self.ds)
def wipedataset(self):
self.ds = SupervisedDataSet(self.inputs, self.outputs)
pass
def cycle(self,action,state):
return self.n.activate([action,state[0],state[1]])
def AddToTrainingSet(self,action,state,output):
out= "New Set","Action: ",action,"State: ", state,"Output: ", output
f.write(str(out)+"\n")
self.ds.addSample((action,state[0],state[1]),output)
def train(self):
return "ERROR",self.trainer.train()
def traintoconverg(self):
x = 10000
y=0
z=100
print len(self.ds),"DS SIZE"
while x > 0.0001 and y < z:
print len(self.ds)
x = self.trainer.train()
print x,"ERROR",y
y+=1
f = open('brains/brain2000.ann','w')
pickle.dump(self.n,f)
def trainfinal(self):
x = 10000
y=0
z=25
while x > 0.00001 and y < z:
x = self.trainer.train()
print x,"ERROR",y
y+=1
def createAndTrainNetworkFromList(train_list,
count_input_samples,
net_filename,
count_layers=33,
count_outputs=1,
max_epochs=15000,
min_epochs=300):
net = buildNetwork(count_input_samples, count_layers, count_outputs)
ds = SupervisedDataSet(count_input_samples, count_outputs)
count_samples = len(train_list)
for i in range(0, count_samples):
ds.addSample(train_list[i][:-count_outputs],
train_list[i][-count_outputs])
trainer = RPropMinusTrainer(net, verbose=True)
trainer.setData(ds)
a = trainer.trainUntilConvergence(maxEpochs=max_epochs,
continueEpochs=min_epochs,
validationProportion=0.15)
net_filename = net_filename[:-4] + str(a[0][-1]) + '.xml'
NetworkWriter.writeToFile(net, net_filename)
result_list = [a, net_filename]
return result_list
def trainNetwork(dirname):
numFeatures = 5000
ds = SequentialDataSet(numFeatures, 1)
tracks = glob.glob(os.path.join(dirname, 'train??.wav'))
for t in tracks:
track = os.path.splitext(t)[0]
# load training data
print "Reading %s..." % track
data = numpy.genfromtxt(track + '_seg.csv', delimiter=",")
labels = numpy.genfromtxt(track + 'REF.txt', delimiter='\t')[0::10,1]
numData = data.shape[0]
# add the input to the dataset
print "Adding to dataset..."
ds.newSequence()
for i in range(numData):
ds.addSample(data[i], (labels[i],))
# initialize the neural network
print "Initializing neural network..."
net = buildNetwork(numFeatures, 50, 1,
hiddenclass=LSTMLayer, outputbias=False, recurrent=True)
# train the network on the dataset
print "Training neural net"
trainer = RPropMinusTrainer(net, dataset=ds)
## trainer.trainUntilConvergence(maxEpochs=50, verbose=True, validationProportion=0.1)
error = -1
for i in range(100):
new_error = trainer.train()
print "error: " + str(new_error)
if abs(error - new_error) < 0.1: break
error = new_error
# save the network
print "Saving neural network..."
NetworkWriter.writeToFile(net, os.path.basename(dirname) + 'net')
def train(self, input_row, output_row):
"""
Training network by r-prop.
PARTITION_OF_EDUCATION_VERIFICATION_SET - education|validation ratio
MAX_EPOCHS - count of max steps of education
OUTCASTING_EPOCHS - if education can't get out of local minimum it given count of steps, it stops
"""
self._form_set(input_row, output_row)
trainer = RPropMinusTrainer(module=self.network, dataset=self.data_set)
self.training_errors, self.validation_errors = trainer.trainUntilConvergence(
validationProportion=self.settings.training_part_fraction,
maxEpochs=self.settings.maximum_training_epochs,
continueEpochs=self.settings.quit_epochs)
len_validate = int(len(output_row[0]['data']) * (1 - self.settings.training_part_fraction))
results_of = [list(self.network.activate(x))[0] for x in self.inputs_for_validation[len_validate:]]
self.mse = sum(map(lambda result, target: fabs(result - target), list(results_of),
list(output_row[0]['data'][len_validate:]))) / len(results_of)
print 'DUMB-dd'
for it in results_of:
print it
print 'DUMB-pp'
for it in list(output_row[0]['data'][len_validate:]):
print it
print '| | |-MSE = ', self.mse
def fit(self, X, y):
"""
Trains the classifier
:param pandas.DataFrame X: data shape [n_samples, n_features]
:param y: labels of events - array-like of shape [n_samples]
.. note::
doesn't support sample weights
"""
dataset = self._prepare_net_and_dataset(X, y, 'classification')
if self.use_rprop:
trainer = RPropMinusTrainer(self.net,
etaminus=self.etaminus,
etaplus=self.etaplus,
deltamin=self.deltamin,
deltamax=self.deltamax,
delta0=self.delta0,
dataset=dataset,
learningrate=self.learningrate,
lrdecay=self.lrdecay,
momentum=self.momentum,
verbose=self.verbose,
batchlearning=self.batchlearning,
weightdecay=self.weightdecay)
else:
trainer = BackpropTrainer(self.net,
dataset,
learningrate=self.learningrate,
lrdecay=self.lrdecay,
momentum=self.momentum,
verbose=self.verbose,
batchlearning=self.batchlearning,
weightdecay=self.weightdecay)
if self.epochs < 0:
trainer.trainUntilConvergence(
maxEpochs=self.max_epochs,
continueEpochs=self.continue_epochs,
verbose=self.verbose,
validationProportion=self.validation_proportion)
else:
for i in range(self.epochs):
trainer.train()
self.__fitted = True
return self
def partial_fit(self, X, y):
"""
Additional training of the estimator
:param pandas.DataFrame X: data shape [n_samples, n_features]
:param y: labels of events - array-like of shape [n_samples]
:return: self
"""
dataset = self._prepare_dataset(X, y, self._model_type)
if not self.is_fitted():
self._prepare_net(dataset=dataset, model_type=self._model_type)
if self.use_rprop:
trainer = RPropMinusTrainer(self.net,
etaminus=self.etaminus,
etaplus=self.etaplus,
deltamin=self.deltamin,
deltamax=self.deltamax,
delta0=self.delta0,
dataset=dataset,
learningrate=self.learningrate,
lrdecay=self.lrdecay,
momentum=self.momentum,
verbose=self.verbose,
batchlearning=self.batchlearning,
weightdecay=self.weightdecay)
else:
trainer = BackpropTrainer(self.net,
dataset,
learningrate=self.learningrate,
lrdecay=self.lrdecay,
momentum=self.momentum,
verbose=self.verbose,
batchlearning=self.batchlearning,
weightdecay=self.weightdecay)
if self.epochs < 0:
trainer.trainUntilConvergence(
maxEpochs=self.max_epochs,
continueEpochs=self.continue_epochs,
verbose=self.verbose,
validationProportion=self.validation_proportion)
else:
trainer.trainEpochs(epochs=self.epochs, )
return self
def trainNetwork(net,
sample_list,
validate_list,
net_filename,
max_epochs=5500,
min_epochs=300):
count_input_samples = len(sample_list)
count_outputs = len(validate_list)
ds = SupervisedDataSet(count_input_samples, count_outputs)
ds.addSample(sample_list, validate_list)
trainer = RPropMinusTrainer(net, verbose=True)
trainer.setData(ds)
trainer.trainUntilConvergence(maxEpochs=max_epochs,
continueEpochs=min_epochs)
NetworkWriter.writeToFile(net, net_filename)
return net
def fit(self, X, y):
"""
Trains the classifier
:param pandas.DataFrame X: data shape [n_samples, n_features]
:param y: labels of events - array-like of shape [n_samples]
.. note::
doesn't support sample weights
"""
dataset = self._prepare_net_and_dataset(X, y, 'classification')
if self.use_rprop:
trainer = RPropMinusTrainer(self.net,
etaminus=self.etaminus,
etaplus=self.etaplus,
deltamin=self.deltamin,
deltamax=self.deltamax,
delta0=self.delta0,
dataset=dataset,
learningrate=self.learningrate,
lrdecay=self.lrdecay,
momentum=self.momentum,
verbose=self.verbose,
batchlearning=self.batchlearning,
weightdecay=self.weightdecay)
else:
trainer = BackpropTrainer(self.net,
dataset,
learningrate=self.learningrate,
lrdecay=self.lrdecay,
momentum=self.momentum,
verbose=self.verbose,
batchlearning=self.batchlearning,
weightdecay=self.weightdecay)
if self.epochs < 0:
trainer.trainUntilConvergence(maxEpochs=self.max_epochs,
continueEpochs=self.continue_epochs,
verbose=self.verbose,
validationProportion=self.validation_proportion)
else:
for i in range(self.epochs):
trainer.train()
self.__fitted = True
return self
def partial_fit(self, X, y):
"""
Additional training of the estimator
:param pandas.DataFrame X: data shape [n_samples, n_features]
:param y: labels of events - array-like of shape [n_samples]
:return: self
"""
dataset = self._prepare_dataset(X, y, self._model_type)
if not self.is_fitted():
self._prepare_net(dataset=dataset, model_type=self._model_type)
if self.use_rprop:
trainer = RPropMinusTrainer(self.net,
etaminus=self.etaminus,
etaplus=self.etaplus,
deltamin=self.deltamin,
deltamax=self.deltamax,
delta0=self.delta0,
dataset=dataset,
learningrate=self.learningrate,
lrdecay=self.lrdecay,
momentum=self.momentum,
verbose=self.verbose,
batchlearning=self.batchlearning,
weightdecay=self.weightdecay)
else:
trainer = BackpropTrainer(self.net,
dataset,
learningrate=self.learningrate,
lrdecay=self.lrdecay,
momentum=self.momentum,
verbose=self.verbose,
batchlearning=self.batchlearning,
weightdecay=self.weightdecay)
if self.epochs < 0:
trainer.trainUntilConvergence(maxEpochs=self.max_epochs,
continueEpochs=self.continue_epochs,
verbose=self.verbose,
validationProportion=self.validation_proportion)
else:
trainer.trainEpochs(epochs=self.epochs, )
return self
tstdata.appendLinked(RopewayJoint[i,:], [5])
trndata._convertToOneOfMany()
tstdata._convertToOneOfMany()
print 'Loaded Dataset!'
#####################
#####################
print 'Building Network'
MLPClassificationNet = buildNetwork(trndata.indim,153,trndata.outdim, hiddenclass=SigmoidLayer, outclass=SoftmaxLayer)
print "Number of weights:",MLPClassificationNet.paramdim
trainer = RPropMinusTrainer(MLPClassificationNet, dataset=trndata, verbose=True, weightdecay=0.01)
tstErrorCount=0
oldtstError=0
trn_error=[]
tst_error=[]
trn_class_accu=[]
tst_class_accu=[]
trnErrorPath='153sigmoid/trn_error'
tstErrorPath='153sigmoid/tst_error'
trnClassErrorPath='153sigmoid/trn_ClassAccu'
tstClassErrorPath='153sigmoid/tst_ClassAccu'
networkPath='153sigmoid/TrainUntilConv.xml'
figPath='153sigmoid/ErrorGraph'
def get_new_trainer(self, data_set):
if not self.neural_net:
self.build_neural_net()
return RPropMinusTrainer(self.neural_net, dataset=data_set)
def trainNetwork(dirname):
numFeatures = 2000
ds = SequentialDataSet(numFeatures, 1)
tracks = glob.glob(os.path.join(dirname, "*.csv"))
for t in tracks:
track = os.path.splitext(t)[0]
# load training data
print "Reading %s..." % t
data = numpy.genfromtxt(t, delimiter=",")
numData = data.shape[0]
# add the input to the dataset
print "Adding to dataset..."
ds.newSequence()
for i in range(numData):
# ds.addSample(data[i], (labels[i],))
input = data[i]
label = input[numFeatures]
if label > 0:
label = midi_util.frequencyToMidi(label)
ds.addSample(input[0:numFeatures], (label,))
# initialize the neural network
print "Initializing neural network..."
# net = buildNetwork(numFeatures, 50, 1,
# hiddenclass=LSTMLayer, bias=True, recurrent=True)
# manual network building
net = RecurrentNetwork()
inlayer = LinearLayer(numFeatures)
# h1 = LSTMLayer(70)
# h2 = SigmoidLayer(50)
octaveLayer = LSTMLayer(5)
noteLayer = LSTMLayer(12)
combinedLayer = SigmoidLayer(60)
outlayer = LinearLayer(1)
net.addInputModule(inlayer)
net.addOutputModule(outlayer)
# net.addModule(h1)
# net.addModule(h2)
net.addModule(octaveLayer)
net.addModule(noteLayer)
net.addModule(combinedLayer)
# net.addConnection(FullConnection(inlayer, h1))
# net.addConnection(FullConnection(h1, h2))
# net.addConnection(FullConnection(h2, outlayer))
net.addConnection(FullConnection(inlayer, octaveLayer))
net.addConnection(FullConnection(inlayer, noteLayer))
# net.addConnection(FullConnection(octaveLayer,combinedLayer))
for i in range(5):
net.addConnection(
FullConnection(
octaveLayer, combinedLayer, inSliceFrom=i, inSliceTo=i + 1, outSliceFrom=i * 12, outSliceTo=(i + 1) * 12
)
)
net.addConnection(FullConnection(noteLayer, combinedLayer))
net.addConnection(FullConnection(combinedLayer, outlayer))
net.sortModules()
# train the network on the dataset
print "Training neural net"
trainer = RPropMinusTrainer(net, dataset=ds)
## trainer.trainUntilConvergence(maxEpochs=50, verbose=True, validationProportion=0.1)
error = -1
for i in range(150):
new_error = trainer.train()
print "error: " + str(new_error)
if abs(error - new_error) < 0.005:
break
error = new_error
# save the network
print "Saving neural network..."
NetworkWriter.writeToFile(net, os.path.basename(dirname) + "designnet")
# build our recurrent network with 10 hidden neurodes, one recurrent
# connection, using tanh activation functions
net = RecurrentNetwork()
hidden_neurodes = 10
net.addInputModule(LinearLayer(len(train_set["input"][0]), name="in"))
net.addModule(TanhLayer(hidden_neurodes, name="hidden1"))
net.addOutputModule(LinearLayer(len(train_set["target"][0]), name="out"))
net.addConnection(FullConnection(net["in"], net["hidden1"], name="c1"))
net.addConnection(FullConnection(net["hidden1"], net["out"], name="c2"))
net.addRecurrentConnection(
FullConnection(net["out"], net["hidden1"], name="cout"))
net.sortModules()
net.randomize()
# train for 30 epochs (overkill) using the rprop- training algorithm
trainer = RPropMinusTrainer(net, dataset=train_set, verbose=True)
trainer.trainOnDataset(train_set, 30)
# test on training set
predictions_train = np.array(
[net.activate(train_set["input"][i])[0] for i in xrange(len(train_set))])
plt.plot(train_set["target"], c="k")
plt.plot(predictions_train, c="r")
plt.show()
# and on test set
predictions_test = np.array(
[net.activate(test_set["input"][i])[0] for i in xrange(len(test_set))])
plt.plot(test_set["target"], c="k")
plt.plot(predictions_test, c="r")
plt.show()
ds = SupervisedDataSet(200 * 60 * 3, 1)
for i in range(len(files)):
img = Image.open("img/" + files[i])
data = np.array(img)
data = data.reshape(-1)
ds.addSample((data), (files[i][0:6]))
img1 = Image.open("test_img/152830.png")
data1 = np.array(img1)
data1 = data1.reshape(-1)
net = buildNetwork(200 * 60 * 3, 1)
trainer = RPropMinusTrainer(net)
trainer.setData(ds)
trainer.trainEpochs(100)
def calculation(a, b):
i = 0
if ((int(a) // 100000) % 10 == (int(b) // 100000) % 10):
i = i + 1
if ((int(a) // 10000) % 10 == (int(b) // 10000) % 10):
i = i + 1
if ((int(a) // 1000) % 10 == (int(b) // 1000) % 10):
i = i + 1
if ((int(a) // 100) % 10 == (int(b) // 100) % 10):
i = i + 1
# one output neuron per class
training_dataset._convertToOneOfMany(bounds=[0, 1])
# same for the independent test data set
testing_dataset = generate_data(test=True)
testing_dataset._convertToOneOfMany(bounds=[0, 1])
# build a feed-forward network with 20 hidden units, plus
# a corresponding trainer
fnn = buildNetwork(training_dataset.indim,
15,
15,
training_dataset.outdim,
outclass=SoftmaxLayer)
#trainer = BackpropTrainer( fnn, dataset=training_dataset,verbose=True)
trainer = RPropMinusTrainer(fnn, dataset=training_dataset, verbose=True)
for i in range(500):
# train the network for 1 epoch
trainer.trainEpochs(15)
# evaluate the result on the training and test data
trnresult = percentError(trainer.testOnClassData(),
training_dataset['class'])
tstresult = percentError(trainer.testOnClassData(dataset=testing_dataset),
testing_dataset['class'])
# print the result
print "epoch: %4d" % trainer.totalepochs, \
" train error: %5.2f%%" % trnresult, \
" test error: %5.2f%%" % tstresult
LSTMClassificationNet.addConnection(FullConnection(LSTMClassificationNet['hidden0'], LSTMClassificationNet['hidden1'], name='h0-to-h1'))
LSTMClassificationNet.addConnection(FullConnection(LSTMClassificationNet['hidden1'], LSTMClassificationNet['out'], name='h1-to-out'))
LSTMClassificationNet.addConnection(FullConnection(LSTMClassificationNet['bias0'], LSTMClassificationNet['hidden0'], name='bias0-to-h0'))
LSTMClassificationNet.addConnection(FullConnection(LSTMClassificationNet['bias1'], LSTMClassificationNet['hidden1'], name='bias1-to-h1'))
LSTMClassificationNet.addRecurrentConnection(FullConnection(LSTMClassificationNet['hidden0'], LSTMClassificationNet['hidden0'], name='h0-to-h0'))
LSTMClassificationNet.addRecurrentConnection(FullConnection(LSTMClassificationNet['hidden1'], LSTMClassificationNet['hidden1'], name='h1-to-h1'))
LSTMClassificationNet.sortModules()
# LSTMClassificationNet = buildNetwork(trndata.indim,20,trndata.outdim, hiddenclass=LSTMLayer,
# outclass=SoftmaxLayer, bias=True, recurrent=True, outputbias=False)
print "Total Number of weights:",LSTMClassificationNet.paramdim
trainer = RPropMinusTrainer(LSTMClassificationNet, dataset=trndata, verbose=True, weightdecay=0.01)
tstErrorCount=0
oldtstError=0
trn_error=[]
tst_error=[]
trn_class_accu=[]
tst_class_accu=[]
trnErrorPath='20LSTMCell/trn_error'
tstErrorPath='20LSTMCell/tst_error'
trnClassErrorPath='20LSTMCell/trn_ClassAccu'
tstClassErrorPath='20LSTMCell/tst_ClassAccu'
networkPath='20LSTMCell/TrainUntilConv.xml'
figPath='20LSTMCell/ErrorGraph'
n.addModule(layer)
n.addOutputModule(outLayer)
n.addConnection(FullConnection(inLayer, hiddenLayers[0]))
for i in range(1, len(hiddenLayers)):
n.addConnection(FullConnection(hiddenLayers[i - 1], hiddenLayers[i]))
n.addConnection(FullConnection(hiddenLayers[len(hiddenLayers) - 1], outLayer))
n.sortModules()
# training set
DS = SupervisedDataSet(10, 5)
trainer = RPropMinusTrainer(n, verbose=True, batchlearning=True, learningrate=0.01, lrdecay=0.0, momentum=0.0,
weightdecay=0.0)
trainer.setData(DS)
data = days('btceUSD.days.csv')
def normalize(v, _max, _min):
return 2.0 / (_max - _min) * (v - _min) - 1.0
def denormalize(v, _max, _min):
return (v + 1.0) / 2.0 * (_max - _min) + _min
def ticks_to_inputs_outputs(ticks, ticks_forecast):
window_prices = map(lambda (x): x[0], ticks)
nr.append(ratio)
print ratio, column
else:
print column, "not an int or long"
return np.array(nr[:-1]), nr[-1]
data = cursor.execute("select %s from adult_data" % columns).fetchall()
dataset = SupervisedDataSet(8, 1)
for row in data:
xd, yd = createNPRow(row)
dataset.addSample(xd, yd)
nn = buildNetwork(8, 3, 1)
trainer = RPropMinusTrainer(nn)
trainer.setData(dataset)
for x in range(5):
error = trainer.train()
print error
errors, success = 0, 0
for row in cursor.execute("select %s from adult_test" % columns).fetchall():
xd, yd = createNPRow(row)
check = int(round(nn.activate(xd[:8])[0]))
if check > 1: check = 1
prediction = possibilities['relation_to_50k_plus'][check]
actual = possibilities['relation_to_50k_plus'][yd]
if prediction == actual:
match = "match"
class NET():
def __init__(self, inputsize, outputsize, hiden=[1]):
self.inputsize = inputsize
self.outputsize = outputsize
self.hiden = hiden
self.err = 1
self.old_err = 1
#print type(self.hiden)
if type(self.hiden) == str:
#print "type str"
self.hiden = self.hiden[1:-1]
b = self.hiden.split(", ")
c = []
for i in b:
c.append(int(i))
self.hiden = c[:]
b = []
b.append(self.inputsize)
b += self.hiden
b.append(self.outputsize)
#print b#"%s, %s, %s, hiddenclass=TanhLayer"%(self.inputsize, self.hiden, self.outputsize)
self.net = FeedForwardNetwork()
self.inputlayer = LinearLayer(self.inputsize, "Input")
self.net.addInputModule(self.inputlayer)
self.outputlayer = LinearLayer(self.outputsize, "Output")
self.net.addOutputModule(self.outputlayer)
self.hidenlayers = []
for i in xrange(len(self.hiden)):
self.hidenlayers.append(SigmoidLayer(self.hiden[i], "hiden%s" % i))
self.net.addModule(self.hidenlayers[-1])
self.net.addConnection(
FullConnection(self.inputlayer, self.outputlayer))
for i in xrange(len(self.hidenlayers)):
self.net.addConnection(
FullConnection(self.inputlayer, self.hidenlayers[i]))
self.net.addConnection(
FullConnection(self.hidenlayers[i], self.outputlayer))
for i in xrange(len(self.hidenlayers)):
for j in xrange(i + 1, len(self.hidenlayers)):
self.net.addConnection(
FullConnection(self.hidenlayers[i], self.hidenlayers[j]))
#self.print_conections(self.net)
self.net.sortModules()
self.ds = SupervisedDataSet(self.inputsize, self.outputsize)
def Update(self, hiden, h):
self.net = FeedForwardNetwork()
self.inputlayer = LinearLayer(self.inputsize, "Input")
self.net.addInputModule(self.inputlayer)
self.outputlayer = LinearLayer(self.outputsize, "Output")
self.net.addOutputModule(self.outputlayer)
self.hidenlayers = []
for i in xrange(len(hiden)):
self.hidenlayers.append(SigmoidLayer(hiden[i], "hiden%s" % i))
self.net.addModule(self.hidenlayers[-1])
self.net.addConnection(
FullConnection(self.inputlayer, self.outputlayer))
for i in xrange(len(self.hidenlayers)):
self.net.addConnection(
FullConnection(self.inputlayer, self.hidenlayers[i]))
self.net.addConnection(
FullConnection(self.hidenlayers[i], self.outputlayer))
for i in xrange(len(self.hidenlayers)):
for j in xrange(i + 1, len(self.hidenlayers)):
if i < h:
self.net.addConnection(
FullConnection(self.hidenlayers[i],
self.hidenlayers[j]))
elif i == h:
self.net.addConnection(
FullConnection(self.hidenlayers[i],
self.hidenlayers[j],
inSliceTo=hiden[i] - 1))
else:
self.net.addConnection(
FullConnection(self.hidenlayers[i],
self.hidenlayers[j]))
#self.print_conections(self.net)
self.net.sortModules()
self.hiden = hiden
def print_conections(self, n):
print("BEGIN")
for mod in n.modules:
print(mod)
for conn in n.connections[mod]:
print(conn)
for cc in range(len(conn.params)):
print(conn.whichBuffers(cc), conn.params[cc])
print("END")
def AddData(self, datainput, dataoutput):
if len(dataoutput) != len(datainput):
print("Not equals data", len(dataoutput), len(datainput))
return 1
self.ds = SupervisedDataSet(self.inputsize, self.outputsize)
for i in xrange(len(dataoutput)):
self.ds.appendLinked(datainput[i], dataoutput[i])
self.trainer = RPropMinusTrainer(self.net,
dataset=self.ds,
learningrate=0.1)
return 0
def AddDataSequential(self, data):
self.ds = SequentialDataSet(self.inputsize, self.outputsize)
for i in xrange(len(data) - 1, 0, -1):
t = data[i]
k = i - 1
while k > -1:
self.ds.appendLinked(data[k], t)
k -= 1
self.ds.newSequence()
"""print self.ds.getNumSequences()
for i in range(self.ds.getNumSequences()):
for input, target in self.ds.getSequenceIterator(i):
print i, TransToIntList_45(input), TransToIntList_45(target)"""
self.trainer = RPropMinusTrainer(self.net,
dataset=self.ds,
learningrate=0.1)
return 0
def TrainNet(self, epoch, error):
if epoch <= 5:
epoch = 5
i = 0
count = 0
while i < epoch:
if error == self.err:
break
self.err = self.trainer.train()
if self.err == self.old_err:
count += 1
else:
count = 0
if count == 3:
self.err = self.old_err
return (self.err, 1)
self.old_err = self.err
i += 1
#self.SaveNet('%s %s_%s_%s.work'%(self.err, self.inputsize, self.hiden, self.outputsize))
return [self.err, 0]
def TrainNetOnce(self):
self.err = self.trainer.train()
return self.err
def SaveNet(self, filename=None):
if filename == None:
NetworkWriter.writeToFile(
self.net, '%s %s_%s_%s.xml' %
(self.err, self.inputsize, self.hiden, self.outputsize))
else:
NetworkWriter.writeToFile(self.net, filename)
def LoadNet(self, fname):
self.net = NetworkReader.readFrom(fname)
tree = ET.parse(fname)
x = tree.getroot()
l = []
for modules in x.findall('Network/Modules/SigmoidLayer/dim'):
l.append(int(modules.get("val")))
self.hiden = l[:]
self.inputsize = self.net.indim
self.outputsize = self.net.outdim
def TestNet(self, inp):
if len(inp) != self.inputsize:
return 0
return self.net.activate(inp[:])
def UpdateWeights(self, f1, f2=None):
n = NetworkReader.readFrom(f1)
if f2 != None:
n2 = NetworkReader.readFrom(f2)
def DictParams(n):
l1 = []
for mod in n.modules:
l = []
for conn in n.connections[mod]:
if conn.paramdim > 0:
l.append([conn.outmod.name, conn.params])
d = dict(l)
l1.append([mod.name, d])
d1 = dict(l1)
return d1
d1 = DictParams(n)
if f2 != None:
d2 = DictParams(n2)
d3 = DictParams(self.net)
params = np.array([])
if f2 != None:
for i in d2:
for j in d2[i]:
b = d3[i][j][:]
b[:d2[i][j].size] = d2[i][j][:]
d3[i].update({j: b})
for i in d1:
for j in d1[i]:
b = d3[i][j][:]
b[:d1[i][j].size] = d1[i][j][:]
d3[i].update({j: b})
for i in d3["Input"]:
params = np.hstack((params, d3["Input"][i]))
for i in xrange(len(self.hiden)):
for j in d3["hiden%s" % i]:
params = np.hstack((params, d3["hiden%s" % i][j]))
self.net._setParameters(params)
def train_nn():
vta = MLFMFCCOnlineAlignedArray(usec0=False, n_last_frames=n_last_frames)
# vta.append_mlf(mlf_sil)
# vta.append_trn(train_data_sil)
vta.append_mlf(mlf_speech)
vta.append_trn(train_data_speech)
mfcc = vta.__iter__().next()
print "MFCC length:", len(mfcc[0])
input_size = len(mfcc[0])
if sigmoid:
net = buildNetwork(input_size,n_hidden_units,n_hidden_units,n_hidden_units,n_hidden_units,2, hiddenclass=SigmoidLayer, outclass=SoftmaxLayer,
bias = True, fast = arac)
else:
net = buildNetwork(input_size,n_hidden_units,n_hidden_units,n_hidden_units,n_hidden_units,2, hiddenclass=TanhLayer, outclass=SoftmaxLayer,
bias = True, fast = arac)
dc_acc = deque(maxlen=20)
dt_acc = deque(maxlen=20)
print "Generating the MFCC features"
vta_new = []
i = 0
for frame, label in vta:
if i % (n_max_frames / 10) == 0:
print "Already processed: %.2f%% of data" % (100.0*i/n_max_frames)
if i > n_max_frames:
break
i += 1
vta_new.append((frame, label))
vta = vta_new
for epoch in range(n_max_epoch):
i = 1
m = 0
ds = SupervisedDataSet(input_size, 2)
c_acc = 0.0
c_sil = 0.0
t_acc = 0.0
t_sil = 0.0
for frame, label in vta:
#print frame
if (i % n_max_frames_per_minibatch) != 0:
if label == "sil":
ds.addSample(frame, (1,0))
else:
ds.addSample(frame, (0,1))
else:
a = net.activateOnDataset(ds)
acc, sil = get_accuracy(ds, a)
print
print "-"*120
if m < n_crossvalid_minibatches:
print "Cross-validation"
c_acc = running_avg(c_acc, m, acc)
c_sil = running_avg(c_sil, m, sil)
else:
print "Training"
t_acc = running_avg(t_acc, m - n_crossvalid_minibatches, acc)
t_sil = running_avg(t_sil, m - n_crossvalid_minibatches, sil)
if bprop:
trainer = BackpropTrainer(net, dataset = ds)
else:
trainer = RPropMinusTrainer(net, dataset = ds)
trainer.train()
m += 1
print
print "n_max_frames, max_files, max_frames_per_segment, trim_segments, n_max_epoch, n_max_frames_per_minibatch, n_hidden_units, sigmoid, arac, n_last_frames, n_crossvalid_minibatches, bprop"
print n_max_frames, max_files, max_frames_per_segment, trim_segments, n_max_epoch, n_max_frames_per_minibatch, n_hidden_units, sigmoid, arac, n_last_frames, n_crossvalid_minibatches, bprop
print "Epoch: %d Mini-batch: %d" % (epoch, m)
print
print "Cross-validation stats"
print "------------------------"
print "Epoch predictive accuracy: %0.2f" % c_acc
print "Last epoch accs:", ["%.2f" % x for x in dc_acc]
print "Epoch sil bias: %0.2f" % c_sil
print
print "Training stats"
print "------------------------"
print "Epoch predictive accuracy: %0.2f" % t_acc
print "Last epoch accs:", ["%.2f" % x for x in dt_acc]
print "Epoch sil bias: %0.2f" % t_sil
print
print "Minibatch stats"
print "------------------------"
print "Mini-batch predictive accuracy: %0.2f" % acc
print "Mini-batch sil bias: %0.2f" % sil
ds = SupervisedDataSet(input_size, 2)
i += 1
dc_acc.append(c_acc)
dt_acc.append(t_acc)
testset.append((a,random.uniform(-math.pi/2,math.pi/2),1)) testset.append((a,random.uniform(-math.pi,-math.pi/2),-1)) testset.append((a,random.uniform(math.pi/2,math.pi),-1)) elif a == 1: testset.append((a,random.uniform(-math.pi,0),1)) testset.append((a,random.uniform(0,math.pi),-1)) else: testset.append((a,random.uniform(0,math.pi),1)) testset.append((a,random.uniform(-math.pi,0),-1)) ann = buildNetwork(2,20,1,bias=True,hiddenclass=TanhLayer) ds = SupervisedDataSet(2,1) ann.sortModules() trainer = RPropMinusTrainer(ann) trainer.setData(ds) for i in dataset: ds.addSample((i[0],i[1]),i[2]) i=10000 x=0 z=100 while i > 0.0001 and x<z: i = trainer.train() print i x+=1 for a in testset: result = ann.activate([a[0],a[1]]) print str(a[2]) +" actual vs. output " +str(result)
def BnTNN(ds, hiddensize, epoches):
net = buildNetwork(ds.indim, hiddensize, ds.outdim, outclass=SoftmaxLayer)
st = time()
trainer = RPropMinusTrainer(net, dataset=ds, momentum=0.1, verbose=True, weightdecay=0.01)#
trainer.trainEpochs( epoches )
return net
class FFNetwork(Network):
def __init__(self, sensor_ids, action_ids, n_hidden, bias=True):
super(FFNetwork, self).__init__(sensor_ids=sensor_ids, action_ids=action_ids)
self.net = buildNetwork(SensorModel.array_length(sensor_ids), n_hidden, 1,
hiddenclass=TanhLayer,
#outclass=TanhLayer,
bias=bias)
self.scaler_input = None
self.trainer = None
def save(self, filename):
with open(filename, 'wb') as f:
pickle.dump(self, f)
def load(self, filename):
with open(filename, 'rb') as f:
ffn = pickle.load(f)
self.net = ffn.net
self.sensor_ids = ffn.sensor_ids
self.action_ids = ffn.action_ids
self.scaler_input = ffn.scaler_input
del ffn
def get_action(self, sensors):
x = sensors.get_array(self.sensor_ids)
if self.scaler_input is not None:
x = self.scaler_input.transform(x)
return self.net.activate(x)[0]
def get_params(self):
pass
def train(self, training_files, learningrate=0.01, scaling=True, noise=False, verbose=True):
print "building dataset..."
ds = SupervisedDataSet(SensorModel.array_length(self.sensor_ids), 1)
# read training file line, create sensormodel object, do backprop
a = None
s = None
for logfile in training_files:
print "loading file", logfile
with open(logfile) as f:
for line in f:
if line.startswith("Received:"):
s = SensorModel(string=line.split(' ', 1)[1])
elif line.startswith("Sending:"):
a = Actions.from_string(string=line.split(' ', 1)[1])
if s is not None and a is not None:
ds.addSample(inp=s.get_array(self.sensor_ids), target=a[self.action_ids[0]])
if noise:
# add the same training sample again but with noise in the sensors
s.add_noise()
ds.addSample(inp=s.get_array(self.sensor_ids), target=a[self.action_ids[0]])
s = None
a = None
print "dataset size:", len(ds)
if scaling:
print "scaling dataset"
self.scaler_input = StandardScaler(with_mean=True, with_std=False).fit(ds.data['input'])
ds.data['input'] = self.scaler_input.transform(ds.data['input'])
ds.data['target'] = ds.data['target']
#self.trainer = BackpropTrainer(self.net, learningrate=learningrate, verbose=verbose)
self.trainer = RPropMinusTrainer(self.net, verbose=verbose, batchlearning=True)
print "training network..."
self.trainer.trainUntilConvergence(dataset=ds, validationProportion=0.25, maxEpochs=10, continueEpochs=2)
ratio = float(float(num) / float(len(possibilities[names[i]])))
nr.append(ratio)
print ratio, column
else:
print column, "not an int or long"
return np.array(nr[:-1]), nr[-1]
data = cursor.execute("select %s from adult_data" % columns).fetchall()
dataset = SupervisedDataSet(8, 1)
for row in data:
xd, yd = createNPRow(row)
dataset.addSample(xd, yd)
nn = buildNetwork(8, 3, 1)
trainer = RPropMinusTrainer(nn)
trainer.setData(dataset)
for x in range(5):
error = trainer.train()
print error
errors, success = 0,0
for row in cursor.execute("select %s from adult_test" % columns).fetchall():
xd, yd = createNPRow(row)
check = int(round(nn.activate(xd[:8])[0]))
if check > 1: check = 1
prediction = possibilities['relation_to_50k_plus'][check]
actual = possibilities['relation_to_50k_plus'][yd]
if prediction == actual:
match = "match"
train_set, test_set = DS.splitWithProportion(0.7) # build our recurrent network with 10 hidden neurodes, one recurrent # connection, using tanh activation functions net = RecurrentNetwork() hidden_neurodes = 10 net.addInputModule(LinearLayer(len(train_set["input"][0]), name="in")) net.addModule(TanhLayer(hidden_neurodes, name="hidden1")) net.addOutputModule(LinearLayer(len(train_set["target"][0]), name="out")) net.addConnection(FullConnection(net["in"], net["hidden1"], name="c1")) net.addConnection(FullConnection(net["hidden1"], net["out"], name="c2")) net.addRecurrentConnection(FullConnection(net["out"], net["hidden1"], name="cout")) net.sortModules() net.randomize() # train for 30 epochs (overkill) using the rprop- training algorithm trainer = RPropMinusTrainer(net, dataset=train_set, verbose=True) trainer.trainOnDataset(train_set, 30) # test on training set predictions_train = np.array([net.activate(train_set["input"][i])[0] for i in xrange(len(train_set))]) plt.plot(train_set["target"], c="k") plt.plot(predictions_train, c="r") plt.show() # and on test set predictions_test = np.array([net.activate(test_set["input"][i])[0] for i in xrange(len(test_set))]) plt.plot(test_set["target"], c="k") plt.plot(predictions_test, c="r") plt.show()
def train_network(options_file_location,training_data_location,output_location):
training_file_handle = open(training_data_location,"r")
training_reader = csv.reader(training_file_handle)
stdout_file = output_location+'training_console_output.txt'
stderr_file = output_location+'training_console_errput.txt'
sys.stdout = open(stdout_file,"w")
sys.stderr = open(stderr_file,"w")
options_file_location = options_file_location
options_file_handle = open(options_file_location,'r')
options_dictionary = {}
for option in options_file_handle.readlines():
key,val = option.split('=')
print key
print val
options_dictionary[key] = val;
num_predictors = int(options_dictionary['num_predictors'])
num_outputs = int(options_dictionary['num_outputs'])
num_training_epochs = int(options_dictionary['num_training_epochs'])
num_hidden_neurons = int(options_dictionary['num_hidden_neurons'])
num_classes = int((options_dictionary['num_classes']))
hidden_neuron_type_str = options_dictionary['hidden_neuron_type']
output_neuron_type_str = options_dictionary['output_neuron_type']
hidden_layer_type,output_layer_type = net_topol.get_layer_types(options_dictionary)
training_dataset = SequenceClassificationDataSet(num_predictors, 1,num_classes)
previous_sequence_number = 1
#read data into dataset objects
print 'reading in training data...'
for row in training_reader:
#convert list of strings to list of floats
list = [float(s) for s in row]
#split input line
predictors = list[0:num_predictors]
#+1 is to skip over the sequence column
outputs = list[num_predictors+1:num_predictors+1+num_outputs]
#convert from python list to numpy array
predictors = np.array(predictors)
outputs = np.array(outputs)
sequence_number = math.trunc(list[num_predictors])
if not sequence_number==previous_sequence_number:
# print sequence_number
# print previous_sequence_number
training_dataset.newSequence()
previous_sequence_number = sequence_number
#add to dataset
training_dataset.appendLinked(predictors, outputs)
network = shortcuts.buildNetwork(num_predictors, num_hidden_neurons, num_outputs, hiddenclass=LSTMLayer, outclass=SoftmaxLayer)
network.sortModules();
training_dataset._convertToOneOfMany();
print str(network)
print str(training_dataset)
trainer = RPropMinusTrainer(module=network, dataset=training_dataset)
for i in range(num_training_epochs):
print 'Starting training epoch: '+str(i)
trainer.trainEpochs(1)
sys.stdout.flush()
network_file_location = output_location+'trained_network.xml'
NetworkWriter.writeToFile(network, network_file_location)
done_file_handle = open(output_location+'training_done.txt',"w")
done_file_handle.write('%s' % 'done!')
done_file_handle.close()
append2DS(DS, snd_18768, 0, nClasses)
# piano
snd_21649 = load_snd(21649)
append2DS(DS, snd_21649, 1, nClasses)
# clock
snd_20043 = load_snd(20043)
append2DS(DS, snd_20043, 2, nClasses)
# fnn = buildNetwork(1, 15, 5, hiddenclass = LSTMLayer, outclass = SoftmaxLayer, outputbias = False, recurrent = True)
fnn = buildNetwork(1, 1, nClasses, hiddenclass=LSTMLayer, outclass=TanhLayer, outputbias=False, recurrent=True)
# Create a trainer for backprop and train the net.
# trainer = BackpropTrainer(fnn, DStrain, learningrate = 0.005)
trainer = RPropMinusTrainer(fnn, dataset=DS, verbose=True)
for i in range(4):
# train the network for 1 epoch
trainer.trainEpochs(1)
print trainer.train()
fnn.reset()
summed = numpy.zeros(nClasses)
for sample in snd_18768:
summed += fnn.activate([sample])
print summed / len(snd_18768)
fnn.reset()
summed = numpy.zeros(nClasses)
for sample in snd_21649:
fnn.sorted = False
fnn.sortModules()
else:
fnn = buildNetwork(
segments,
nHidden,
segments,
hiddenclass=LSTMLayer,
outclass=TanhLayer,
outputbias=True,
recurrent=True,
peepholes=False,
fast=False,
)
trainer = RPropMinusTrainer(fnn, dataset=DS, verbose=True)
print "Begin training..."
# Train the network
trainer.trainEpochs(nEpoch)
# Store the encoder
file = open("autoencoder-%i.xml" % (nHidden), "w")
pickle.dump(fnn, file)
file.close()
# Show an example
id = 86464
cochleogram = data.load_cochleogram(id)
fnn.reset()
training_dataset = generate_data()
# neural networks work better if classes are encoded using
# one output neuron per class
training_dataset._convertToOneOfMany( bounds=[0,1] )
# same for the independent test data set
testing_dataset = generate_data(test=True)
testing_dataset._convertToOneOfMany( bounds=[0,1] )
# build a feed-forward network with 20 hidden units, plus
# a corresponding trainer
# fnn = buildNetwork( training_dataset.indim, 15,15, training_dataset.outdim, outclass=SoftmaxLayer )
fnn = buildNetwork( training_dataset.indim, 15, training_dataset.outdim, hiddenclass=LSTMLayer, outclass=SoftmaxLayer, outputbias=False, recurrent=True)
trainer = RPropMinusTrainer( fnn, dataset=training_dataset, verbose=True )
#trainer = BackpropTrainer( fnn, dataset=training_dataset,verbose=True)
for i in range(500):
# train the network for 1 epoch
trainer.trainEpochs( 15 )
# evaluate the result on the training and test data
trnresult = percentError( trainer.testOnClassData(),
training_dataset['class'] )
tstresult = percentError( trainer.testOnClassData(
dataset=testing_dataset ), testing_dataset['class'] )
# print the result
print "epoch: %4d" % trainer.totalepochs, \
" train error: %5.2f%%" % trnresult, \
arg = (i / n) * 6 - 3
x.append(arg)
r = f(arg) + (random() - 0.5) * 0.2
y.append(f(arg))
y_noise.append(r)
ds.addSample((arg), (r))
trainer_big = BackpropTrainer(net_big,
ds,
learningrate=0.01,
lrdecay=1.0,
momentum=0.0,
weightdecay=0.0)
# RProp-, cf. [Igel&Huesken, Neurocomputing 50, 2003
trainer = RPropMinusTrainer(net, dataset=ds)
# trainer.trainUntilConvergence()
for i in range(100):
trainer.train()
for i in range(10):
trainer_big.train()
for i in range(n):
arg = (i / n) * 6 - 3
y_n.append(net.activate([arg]))
y_n_big.append(net_big.activate([arg]))
fig = plt.figure()
def recurrent_neural_network(self,
train_X,
train_y,
test_X,
test_y,
n_hidden_neurons=50,
iterations=100,
gridsearch=False,
gridsearch_training_frac=0.7,
outputbias=False,
error='accuracy'):
if gridsearch:
n_hidden_neurons, iterations, outputbias = self.gridsearch_recurrent_neural_network(
train_X,
train_y,
test_X,
test_y,
gridsearch_training_frac=gridsearch_training_frac,
error=error)
# Create numerical datasets first.
new_train_X, new_test_X = self.create_numerical_multiple_dataset(
train_X, test_X)
new_train_y, new_test_y = self.create_numerical_multiple_dataset(
train_y, test_y)
# We normalize the input.....
new_train_X, new_test_X, min_X, max_X = self.normalize(
new_train_X, new_test_X, 0, 1)
new_train_y, new_test_y, min_y, max_y = self.normalize(
new_train_y, new_test_y, 0.1, 0.9)
# Create the proper pybrain datasets.
ds_training = self.rnn_dataset(new_train_X, new_train_y)
ds_test = self.rnn_dataset(new_test_X, new_test_y)
inputs = len(new_train_X.columns)
outputs = len(new_train_y.columns)
# Build the network with the proper parameters.
n = buildNetwork(inputs,
n_hidden_neurons,
outputs,
hiddenclass=SigmoidLayer,
outclass=SigmoidLayer,
outputbias=outputbias,
recurrent=True)
# Train using back propagation through time.
#trainer = BackpropTrainer(n, dataset=ds_training, verbose=False, momentum=0.9, learningrate=0.01)
trainer = RPropMinusTrainer(n, dataset=ds_training, verbose=False)
for i in range(0, iterations):
trainer.train()
# for mod in n.modules:
# for conn in n.connections[mod]:
# print conn
# for cc in range(len(conn.params)):
# print conn.whichBuffers(cc), conn.params[cc]
# Determine performance on the training and test set....
# Y_train = []
# for i in range(0, len(new_train_X.index)):
# input = tuple(new_train_X.ix[i,:].values)
# output = n.activate(input)
# Y_train.append(output)
# Y_test = []
# for i in range(0, len(new_test_X.index)):
# Y_test.append(n.activate(tuple(new_test_X.ix[i,:].values)))
Y_train = []
Y_test = []
for sample, target in ds_training.getSequenceIterator(0):
Y_train.append(n.activate(sample).tolist())
for sample, target in ds_test.getSequenceIterator(0):
Y_test.append(n.activate(sample).tolist())
y_train_result = pd.DataFrame(Y_train,
columns=new_train_y.columns,
index=train_y.index)
y_test_result = pd.DataFrame(Y_test,
columns=new_test_y.columns,
index=test_y.index)
# print y_train_result
y_train_result = self.denormalize(y_train_result, min_y, max_y, 0.1,
0.9)
y_test_result = self.denormalize(y_test_result, min_y, max_y, 0.1, 0.9)
# plot.plot(train_y.index, train_y)
# plot.hold(True)
# plot.plot(train_y.index, pred_train_y_prob)
# plot.show()
return y_train_result.idxmax(axis=1), y_test_result.idxmax(
axis=1), y_train_result, y_test_result
def train(self, epoch):
self.ds._convertToOneOfMany( )
trainer = RPropMinusTrainer(self.net, dataset=self.ds, momentum=0.1, verbose=True, weightdecay=0.01)
trainer.trainEpochs( epoch )
