def recurrent_neural_network(self, train_X, train_y, test_X, test_y, n_hidden_neurons=50, iterations=100,
gridsearch=False, gridsearch_training_frac=0.7, outputbias=False, error='accuracy'):
"""
Apply a recurrent neural network for classification upon the training data (with the specified number of
hidden neurons and iterations), and use the created network to predict the outcome for both the test and
training set. It returns the categorical predictions for the training and test set as well as the
probabilities associated with each class, each class being represented as a column in the data frame.
"""
if gridsearch:
n_hidden_neurons, iterations, outputbias = self. \
gridsearch_recurrent_neural_network(train_X, train_y, gridsearch_training_frac=gridsearch_training_frac,
error=error)
# Create numerical datasets first
new_train_X, new_test_X = self.create_numerical_multiple_dataset(train_X, test_X)
new_train_y, new_test_y = self.create_numerical_multiple_dataset(train_y, test_y)
# Normalize the input
new_train_X, new_test_X, min_X, max_X = self.normalize(new_train_X, new_test_X, 0, 1)
new_train_y, new_test_y, min_y, max_y = self.normalize(new_train_y, new_test_y, 0.1, 0.9)
# Create the proper pybrain datasets
ds_training = self.rnn_dataset(new_train_X, new_train_y)
ds_test = self.rnn_dataset(new_test_X, new_test_y)
inputs = len(new_train_X.columns)
outputs = len(new_train_y.columns)
# Build the network with the proper parameters
n = buildNetwork(inputs, n_hidden_neurons, outputs, hiddenclass=SigmoidLayer, outclass=SigmoidLayer,
outputbias=outputbias, recurrent=True)
# Train using back propagation through time
# trainer = BackpropTrainer(n, dataset=ds_training, verbose=False, momentum=0.9, learningrate=0.01)
trainer = RPropMinusTrainer(n, dataset=ds_training, verbose=False)
for i in range(0, iterations):
trainer.train()
Y_train = []
Y_test = []
for sample, target in ds_training.getSequenceIterator(0):
Y_train.append(n.activate(sample).tolist())
for sample, target in ds_test.getSequenceIterator(0):
Y_test.append(n.activate(sample).tolist())
y_train_result = pd.DataFrame(Y_train, columns=new_train_y.columns, index=train_y.index)
y_test_result = pd.DataFrame(Y_test, columns=new_test_y.columns, index=test_y.index)
y_train_result = self.denormalize(y_train_result, min_y, max_y, 0.1, 0.9)
y_test_result = self.denormalize(y_test_result, min_y, max_y, 0.1, 0.9)
return y_train_result.idxmax(axis=1), y_test_result.idxmax(axis=1), y_train_result, y_test_result
def fit(self, X, y):
"""
Trains the classifier
:param pandas.DataFrame X: data shape [n_samples, n_features]
:param y: labels of events - array-like of shape [n_samples]
.. note::
doesn't support sample weights
"""
dataset = self._prepare_net_and_dataset(X, y, 'classification')
if self.use_rprop:
trainer = RPropMinusTrainer(self.net,
etaminus=self.etaminus,
etaplus=self.etaplus,
deltamin=self.deltamin,
deltamax=self.deltamax,
delta0=self.delta0,
dataset=dataset,
learningrate=self.learningrate,
lrdecay=self.lrdecay,
momentum=self.momentum,
verbose=self.verbose,
batchlearning=self.batchlearning,
weightdecay=self.weightdecay)
else:
trainer = BackpropTrainer(self.net,
dataset,
learningrate=self.learningrate,
lrdecay=self.lrdecay,
momentum=self.momentum,
verbose=self.verbose,
batchlearning=self.batchlearning,
weightdecay=self.weightdecay)
if self.epochs < 0:
trainer.trainUntilConvergence(
maxEpochs=self.max_epochs,
continueEpochs=self.continue_epochs,
verbose=self.verbose,
validationProportion=self.validation_proportion)
else:
for i in range(self.epochs):
trainer.train()
self.__fitted = True
return self
def train(
train,
label,
custom_net=None,
training_mse_threshold=0.40,
testing_mse_threshold=0.60,
epoch_threshold=10,
epochs=100,
hidden_size=20,
):
# Test Set.
x_train = train[0:split_at, :]
y_train_slice = label.__getslice__(0, split_at)
y_train = y_train_slice.reshape(-1, 1)
x_test = train[split_at:, :]
y_test_slice = label.__getslice__(split_at, label.shape[0])
y_test = y_test_slice.reshape(-1, 1)
# Shape.
input_size = x_train.shape[1]
target_size = y_train.shape[1]
# prepare dataset
ds = SDS(input_size, target_size)
ds.setField("input", x_train)
ds.setField("target", y_train)
# prepare dataset
ds_test = SDS(input_size, target_size)
ds_test.setField("input", x_test)
ds_test.setField("target", y_test)
min_mse = 1000000
# init and train
if custom_net == None:
net = buildNetwork(input_size, hidden_size, target_size, bias=True)
else:
print "Picking up the custom network"
net = custom_net
trainer = RPropMinusTrainer(net, dataset=ds, verbose=False, weightdecay=0.01, batchlearning=True)
print "training for {} epochs...".format(epochs)
for i in range(epochs):
mse = trainer.train()
print "training mse, epoch {}: {}".format(i + 1, math.sqrt(mse))
p = net.activateOnDataset(ds_test)
mse = math.sqrt(MSE(y_test, p))
print "-- testing mse, epoch {}: {}".format(i + 1, mse)
pickle.dump(net, open("current_run", "wb"))
if min_mse > mse:
print "Current minimum found at ", i
pickle.dump(net, open("current_min_epoch_" + model_file, "wb"))
min_mse = mse
pickle.dump(net, open(model_file, "wb"))
return net
def fit(self, X, y):
"""
Trains the classifier
:param pandas.DataFrame X: data shape [n_samples, n_features]
:param y: labels of events - array-like of shape [n_samples]
.. note::
doesn't support sample weights
"""
dataset = self._prepare_net_and_dataset(X, y, 'classification')
if self.use_rprop:
trainer = RPropMinusTrainer(self.net,
etaminus=self.etaminus,
etaplus=self.etaplus,
deltamin=self.deltamin,
deltamax=self.deltamax,
delta0=self.delta0,
dataset=dataset,
learningrate=self.learningrate,
lrdecay=self.lrdecay,
momentum=self.momentum,
verbose=self.verbose,
batchlearning=self.batchlearning,
weightdecay=self.weightdecay)
else:
trainer = BackpropTrainer(self.net,
dataset,
learningrate=self.learningrate,
lrdecay=self.lrdecay,
momentum=self.momentum,
verbose=self.verbose,
batchlearning=self.batchlearning,
weightdecay=self.weightdecay)
if self.epochs < 0:
trainer.trainUntilConvergence(maxEpochs=self.max_epochs,
continueEpochs=self.continue_epochs,
verbose=self.verbose,
validationProportion=self.validation_proportion)
else:
for i in range(self.epochs):
trainer.train()
self.__fitted = True
return self
def train(self, trndata, valdata, hidden_neurons=5, hidden_class=SigmoidLayer, iterations=3):
print "Hidden neurons: " + str(hidden_neurons)
print "Hidden class: " + str(hidden_class)
print "Iterations: " + str(iterations)
fnn = buildNetwork(trndata.indim, hidden_neurons, trndata.outdim, outclass=SoftmaxLayer,
hiddenclass=hidden_class)
trainer = RPropMinusTrainer(fnn, dataset=trndata, verbose=False)
#trainer = BackpropTrainer(fnn, dataset=trndata, momentum=0.5, verbose=True, learningrate=0.05)
for i in range(iterations):
trainer.train()
out, tar = trainer.testOnClassData(dataset=valdata, return_targets=True, verbose=False)
#used to return final score, not used yet :D
print str(i) + " " + str(accuracy(out, tar))
self.model = trainer
def train_net():
fnn = buildNetwork(len(input_args), 3, 2)
ds = ClassificationDataSet(len(input_args),2,nb_classes=2)
ds = generate_data(ds , hour_to_use_app = 10)
trainer = RPropMinusTrainer( fnn, dataset= ds, verbose=True)
trainer.train()
trainer.trainEpochs(15)
test = ClassificationDataSet(4,2)
test.addSample((12,6,10,6),[1,0])
test.addSample((12,1,7,2),[0,1])
test.addSample((12,3,11,1),[0,1])
fnn.activateOnDataset(test)
return fnn,trainer,ds,test
def train_cross_validate(train, label, custom_net=None, training_mse_threshold=0.40, testing_mse_threshold=0.60,
epoch_threshold=10, epochs=100, hidden_size=50):
# Test Set.
x_train = train[0:split_at, :]
y_train_slice = label.__getslice__(0, split_at)
y_train = y_train_slice.reshape(-1, 1)
x_test = train[split_at:, :]
y_test_slice = label.__getslice__(split_at, label.shape[0])
y_test = y_test_slice.reshape(-1, 1)
# Shape.
input_size = x_train.shape[1]
target_size = y_train.shape[1]
input_size_test = x_test.shape[1]
target_size_test = y_test.shape[1]
# prepare dataset
ds = SDS(input_size, target_size)
ds.setField('input', x_train)
ds.setField('target', y_train)
# prepare dataset
ds_test = SDS(input_size, target_size)
ds_test.setField('input', x_test)
ds_test.setField('target', y_test)
min_mse = 1000000
# init and train
if custom_net == None:
net = buildNetwork(input_size, hidden_size, target_size, bias=True, hiddenclass=TanhLayer)
else:
print "Picking up the custom network"
net = custom_net
trainer = RPropMinusTrainer(net, dataset=ds, verbose=True, weightdecay=0.01, batchlearning=True)
print "training for {} epochs...".format(epochs)
for i in range(epochs):
mse = trainer.train()
print "training mse, epoch {}: {}".format(i + 1, mse)
p = net.activateOnDataset(ds_test)
mse = MSE(y_test, p)
print "-- testing mse, epoch {}: {}".format(i + 1, mse)
pickle.dump(net, open("current_run", 'wb'))
if min_mse > mse:
print "Current minimum found at ", i
pickle.dump(net, open("current_min_epoch_" + model_file, 'wb'))
min_mse = mse
pickle.dump(net, open(model_file, 'wb'))
return net
def train(self,
cycles,
percent,
hidden_layers=3,
hiddenclass=None,
num_outputs=1,
num_inputs=-1):
num_inputs = self._count_inputs() if num_inputs == -1 else num_inputs
if num_inputs <= 0 or num_outputs <= 0 or cycles <= 0 or (percent > 100 or percent <= 0):
return
network = self._buildNet(hidden_layers, num_outputs, num_inputs, hiddenclass)
data_set = self.get_data_set(percent, num_inputs, num_outputs)
trainer = RPropMinusTrainer(network, dataset=data_set)
for i in range(cycles):
trainer.train()
return network
class Brain():
def __init__(self):
self.inputs = 3
self.outputs = 1
self.n = buildNetwork(self.inputs, 200,200,200,200,self.outputs, bias=True,hiddenclass=TanhLayer)
self.n.sortModules()
self.ds = SupervisedDataSet(self.inputs, self.outputs)
self.trainer = RPropMinusTrainer(self.n)
self.trainer.setData(self.ds)
def wipedataset(self):
self.ds = SupervisedDataSet(self.inputs, self.outputs)
pass
def cycle(self,action,state):
return self.n.activate([action,state[0],state[1]])
def AddToTrainingSet(self,action,state,output):
out= "New Set","Action: ",action,"State: ", state,"Output: ", output
f.write(str(out)+"\n")
self.ds.addSample((action,state[0],state[1]),output)
def train(self):
return "ERROR",self.trainer.train()
def traintoconverg(self):
x = 10000
y=0
z=100
print len(self.ds),"DS SIZE"
while x > 0.0001 and y < z:
print len(self.ds)
x = self.trainer.train()
print x,"ERROR",y
y+=1
f = open('brains/brain2000.ann','w')
pickle.dump(self.n,f)
def trainfinal(self):
x = 10000
y=0
z=25
while x > 0.00001 and y < z:
x = self.trainer.train()
print x,"ERROR",y
y+=1
def trainNetwork(dirname):
numFeatures = 5000
ds = SequentialDataSet(numFeatures, 1)
tracks = glob.glob(os.path.join(dirname, 'train??.wav'))
for t in tracks:
track = os.path.splitext(t)[0]
# load training data
print "Reading %s..." % track
data = numpy.genfromtxt(track + '_seg.csv', delimiter=",")
labels = numpy.genfromtxt(track + 'REF.txt', delimiter='\t')[0::10,1]
numData = data.shape[0]
# add the input to the dataset
print "Adding to dataset..."
ds.newSequence()
for i in range(numData):
ds.addSample(data[i], (labels[i],))
# initialize the neural network
print "Initializing neural network..."
net = buildNetwork(numFeatures, 50, 1,
hiddenclass=LSTMLayer, outputbias=False, recurrent=True)
# train the network on the dataset
print "Training neural net"
trainer = RPropMinusTrainer(net, dataset=ds)
## trainer.trainUntilConvergence(maxEpochs=50, verbose=True, validationProportion=0.1)
error = -1
for i in range(100):
new_error = trainer.train()
print "error: " + str(new_error)
if abs(error - new_error) < 0.1: break
error = new_error
# save the network
print "Saving neural network..."
NetworkWriter.writeToFile(net, os.path.basename(dirname) + 'net')
elif a == 1: testset.append((a,random.uniform(-math.pi,0),1)) testset.append((a,random.uniform(0,math.pi),-1)) else: testset.append((a,random.uniform(0,math.pi),1)) testset.append((a,random.uniform(-math.pi,0),-1)) ann = buildNetwork(2,20,1,bias=True,hiddenclass=TanhLayer) ds = SupervisedDataSet(2,1) ann.sortModules() trainer = RPropMinusTrainer(ann) trainer.setData(ds) for i in dataset: ds.addSample((i[0],i[1]),i[2]) i=10000 x=0 z=100 while i > 0.0001 and x<z: i = trainer.train() print i x+=1 for a in testset: result = ann.activate([a[0],a[1]]) print str(a[2]) +" actual vs. output " +str(result)
def trainNetwork(dirname):
numFeatures = 2000
ds = SequentialDataSet(numFeatures, 1)
tracks = glob.glob(os.path.join(dirname, "*.csv"))
for t in tracks:
track = os.path.splitext(t)[0]
# load training data
print "Reading %s..." % t
data = numpy.genfromtxt(t, delimiter=",")
numData = data.shape[0]
# add the input to the dataset
print "Adding to dataset..."
ds.newSequence()
for i in range(numData):
# ds.addSample(data[i], (labels[i],))
input = data[i]
label = input[numFeatures]
if label > 0:
label = midi_util.frequencyToMidi(label)
ds.addSample(input[0:numFeatures], (label,))
# initialize the neural network
print "Initializing neural network..."
# net = buildNetwork(numFeatures, 50, 1,
# hiddenclass=LSTMLayer, bias=True, recurrent=True)
# manual network building
net = RecurrentNetwork()
inlayer = LinearLayer(numFeatures)
# h1 = LSTMLayer(70)
# h2 = SigmoidLayer(50)
octaveLayer = LSTMLayer(5)
noteLayer = LSTMLayer(12)
combinedLayer = SigmoidLayer(60)
outlayer = LinearLayer(1)
net.addInputModule(inlayer)
net.addOutputModule(outlayer)
# net.addModule(h1)
# net.addModule(h2)
net.addModule(octaveLayer)
net.addModule(noteLayer)
net.addModule(combinedLayer)
# net.addConnection(FullConnection(inlayer, h1))
# net.addConnection(FullConnection(h1, h2))
# net.addConnection(FullConnection(h2, outlayer))
net.addConnection(FullConnection(inlayer, octaveLayer))
net.addConnection(FullConnection(inlayer, noteLayer))
# net.addConnection(FullConnection(octaveLayer,combinedLayer))
for i in range(5):
net.addConnection(
FullConnection(
octaveLayer, combinedLayer, inSliceFrom=i, inSliceTo=i + 1, outSliceFrom=i * 12, outSliceTo=(i + 1) * 12
)
)
net.addConnection(FullConnection(noteLayer, combinedLayer))
net.addConnection(FullConnection(combinedLayer, outlayer))
net.sortModules()
# train the network on the dataset
print "Training neural net"
trainer = RPropMinusTrainer(net, dataset=ds)
## trainer.trainUntilConvergence(maxEpochs=50, verbose=True, validationProportion=0.1)
error = -1
for i in range(150):
new_error = trainer.train()
print "error: " + str(new_error)
if abs(error - new_error) < 0.005:
break
error = new_error
# save the network
print "Saving neural network..."
NetworkWriter.writeToFile(net, os.path.basename(dirname) + "designnet")
return np.array(nr[:-1]), nr[-1]
data = cursor.execute("select %s from adult_data" % columns).fetchall()
dataset = SupervisedDataSet(8, 1)
for row in data:
xd, yd = createNPRow(row)
dataset.addSample(xd, yd)
nn = buildNetwork(8, 3, 1)
trainer = RPropMinusTrainer(nn)
trainer.setData(dataset)
for x in range(5):
error = trainer.train()
print error
errors, success = 0, 0
for row in cursor.execute("select %s from adult_test" % columns).fetchall():
xd, yd = createNPRow(row)
check = int(round(nn.activate(xd[:8])[0]))
if check > 1: check = 1
prediction = possibilities['relation_to_50k_plus'][check]
actual = possibilities['relation_to_50k_plus'][yd]
if prediction == actual:
match = "match"
success += 1
else:
match = "no match"
errors += 1
class NET():
def __init__(self, inputsize, outputsize, hiden=[1]):
self.inputsize = inputsize
self.outputsize = outputsize
self.hiden = hiden
self.err = 1
self.old_err = 1
#print type(self.hiden)
if type(self.hiden) == str:
#print "type str"
self.hiden = self.hiden[1:-1]
b = self.hiden.split(", ")
c = []
for i in b:
c.append(int(i))
self.hiden = c[:]
b = []
b.append(self.inputsize)
b += self.hiden
b.append(self.outputsize)
#print b#"%s, %s, %s, hiddenclass=TanhLayer"%(self.inputsize, self.hiden, self.outputsize)
self.net = FeedForwardNetwork()
self.inputlayer = LinearLayer(self.inputsize, "Input")
self.net.addInputModule(self.inputlayer)
self.outputlayer = LinearLayer(self.outputsize, "Output")
self.net.addOutputModule(self.outputlayer)
self.hidenlayers = []
for i in xrange(len(self.hiden)):
self.hidenlayers.append(SigmoidLayer(self.hiden[i], "hiden%s" % i))
self.net.addModule(self.hidenlayers[-1])
self.net.addConnection(
FullConnection(self.inputlayer, self.outputlayer))
for i in xrange(len(self.hidenlayers)):
self.net.addConnection(
FullConnection(self.inputlayer, self.hidenlayers[i]))
self.net.addConnection(
FullConnection(self.hidenlayers[i], self.outputlayer))
for i in xrange(len(self.hidenlayers)):
for j in xrange(i + 1, len(self.hidenlayers)):
self.net.addConnection(
FullConnection(self.hidenlayers[i], self.hidenlayers[j]))
#self.print_conections(self.net)
self.net.sortModules()
self.ds = SupervisedDataSet(self.inputsize, self.outputsize)
def Update(self, hiden, h):
self.net = FeedForwardNetwork()
self.inputlayer = LinearLayer(self.inputsize, "Input")
self.net.addInputModule(self.inputlayer)
self.outputlayer = LinearLayer(self.outputsize, "Output")
self.net.addOutputModule(self.outputlayer)
self.hidenlayers = []
for i in xrange(len(hiden)):
self.hidenlayers.append(SigmoidLayer(hiden[i], "hiden%s" % i))
self.net.addModule(self.hidenlayers[-1])
self.net.addConnection(
FullConnection(self.inputlayer, self.outputlayer))
for i in xrange(len(self.hidenlayers)):
self.net.addConnection(
FullConnection(self.inputlayer, self.hidenlayers[i]))
self.net.addConnection(
FullConnection(self.hidenlayers[i], self.outputlayer))
for i in xrange(len(self.hidenlayers)):
for j in xrange(i + 1, len(self.hidenlayers)):
if i < h:
self.net.addConnection(
FullConnection(self.hidenlayers[i],
self.hidenlayers[j]))
elif i == h:
self.net.addConnection(
FullConnection(self.hidenlayers[i],
self.hidenlayers[j],
inSliceTo=hiden[i] - 1))
else:
self.net.addConnection(
FullConnection(self.hidenlayers[i],
self.hidenlayers[j]))
#self.print_conections(self.net)
self.net.sortModules()
self.hiden = hiden
def print_conections(self, n):
print("BEGIN")
for mod in n.modules:
print(mod)
for conn in n.connections[mod]:
print(conn)
for cc in range(len(conn.params)):
print(conn.whichBuffers(cc), conn.params[cc])
print("END")
def AddData(self, datainput, dataoutput):
if len(dataoutput) != len(datainput):
print("Not equals data", len(dataoutput), len(datainput))
return 1
self.ds = SupervisedDataSet(self.inputsize, self.outputsize)
for i in xrange(len(dataoutput)):
self.ds.appendLinked(datainput[i], dataoutput[i])
self.trainer = RPropMinusTrainer(self.net,
dataset=self.ds,
learningrate=0.1)
return 0
def AddDataSequential(self, data):
self.ds = SequentialDataSet(self.inputsize, self.outputsize)
for i in xrange(len(data) - 1, 0, -1):
t = data[i]
k = i - 1
while k > -1:
self.ds.appendLinked(data[k], t)
k -= 1
self.ds.newSequence()
"""print self.ds.getNumSequences()
for i in range(self.ds.getNumSequences()):
for input, target in self.ds.getSequenceIterator(i):
print i, TransToIntList_45(input), TransToIntList_45(target)"""
self.trainer = RPropMinusTrainer(self.net,
dataset=self.ds,
learningrate=0.1)
return 0
def TrainNet(self, epoch, error):
if epoch <= 5:
epoch = 5
i = 0
count = 0
while i < epoch:
if error == self.err:
break
self.err = self.trainer.train()
if self.err == self.old_err:
count += 1
else:
count = 0
if count == 3:
self.err = self.old_err
return (self.err, 1)
self.old_err = self.err
i += 1
#self.SaveNet('%s %s_%s_%s.work'%(self.err, self.inputsize, self.hiden, self.outputsize))
return [self.err, 0]
def TrainNetOnce(self):
self.err = self.trainer.train()
return self.err
def SaveNet(self, filename=None):
if filename == None:
NetworkWriter.writeToFile(
self.net, '%s %s_%s_%s.xml' %
(self.err, self.inputsize, self.hiden, self.outputsize))
else:
NetworkWriter.writeToFile(self.net, filename)
def LoadNet(self, fname):
self.net = NetworkReader.readFrom(fname)
tree = ET.parse(fname)
x = tree.getroot()
l = []
for modules in x.findall('Network/Modules/SigmoidLayer/dim'):
l.append(int(modules.get("val")))
self.hiden = l[:]
self.inputsize = self.net.indim
self.outputsize = self.net.outdim
def TestNet(self, inp):
if len(inp) != self.inputsize:
return 0
return self.net.activate(inp[:])
def UpdateWeights(self, f1, f2=None):
n = NetworkReader.readFrom(f1)
if f2 != None:
n2 = NetworkReader.readFrom(f2)
def DictParams(n):
l1 = []
for mod in n.modules:
l = []
for conn in n.connections[mod]:
if conn.paramdim > 0:
l.append([conn.outmod.name, conn.params])
d = dict(l)
l1.append([mod.name, d])
d1 = dict(l1)
return d1
d1 = DictParams(n)
if f2 != None:
d2 = DictParams(n2)
d3 = DictParams(self.net)
params = np.array([])
if f2 != None:
for i in d2:
for j in d2[i]:
b = d3[i][j][:]
b[:d2[i][j].size] = d2[i][j][:]
d3[i].update({j: b})
for i in d1:
for j in d1[i]:
b = d3[i][j][:]
b[:d1[i][j].size] = d1[i][j][:]
d3[i].update({j: b})
for i in d3["Input"]:
params = np.hstack((params, d3["Input"][i]))
for i in xrange(len(self.hiden)):
for j in d3["hiden%s" % i]:
params = np.hstack((params, d3["hiden%s" % i][j]))
self.net._setParameters(params)
arg = (i/n)*6 - 3
x.append(arg)
r = f(arg) + (random() - 0.5) * 0.2
y.append(f(arg))
y_noise.append(r)
ds.addSample((arg), (r))
trainer_big = BackpropTrainer(net_big, ds, learningrate=0.01, lrdecay=1.0, momentum=0.0, weightdecay=0.0)
# RProp-, cf. [Igel&Huesken, Neurocomputing 50, 2003
trainer = RPropMinusTrainer(net, dataset=ds)
# trainer.trainUntilConvergence()
for i in range(100):
trainer.train()
for i in range(10):
trainer_big.train()
for i in range(n):
arg = (i/n)*6 - 3
y_n.append(net.activate([arg]))
y_n_big.append(net_big.activate([arg]))
fig = plt.figure()
plt.plot(x, y, 'black')
plt.plot(x, y_noise, 'r')
plt.plot(x, y_n, 'b')
print column, "not an int or long"
return np.array(nr[:-1]), nr[-1]
data = cursor.execute("select %s from adult_data" % columns).fetchall()
dataset = SupervisedDataSet(8, 1)
for row in data:
xd, yd = createNPRow(row)
dataset.addSample(xd, yd)
nn = buildNetwork(8, 3, 1)
trainer = RPropMinusTrainer(nn)
trainer.setData(dataset)
for x in range(5):
error = trainer.train()
print error
errors, success = 0,0
for row in cursor.execute("select %s from adult_test" % columns).fetchall():
xd, yd = createNPRow(row)
check = int(round(nn.activate(xd[:8])[0]))
if check > 1: check = 1
prediction = possibilities['relation_to_50k_plus'][check]
actual = possibilities['relation_to_50k_plus'][yd]
if prediction == actual:
match = "match"
success += 1
else:
match = "no match"
errors += 1
eta=(total*epochCount/epochs)-total
tm=eta
tm=int(tm/1000)
s=str(tm%60).rjust(2,'0')
tm=int(tm/60)
m=str(tm%60).rjust(2,'0')
tm=int(tm/60)
h=str(tm%24).rjust(2,'0')
tm=int(tm/24)
d=str(tm).rjust(3,'0')
cap=" ETA: %sd%sh%sm%ss" % (d,h,m,s)
text=font.render(cap,True,(255,255,255))
visual.blit(text,(4,44))
n.reset() # reset to t0 for training sequence
emse=trainer.train()
if rendering:
xOff=4+11*23
eRect=pygame.Rect(xOff,44,11*17,20)
pygame.draw.rect(visual,(96,96,96),eRect)
frac=str(emse).split('.')
if len(frac)<2:
frac.append('')
(w,f)=frac
w=w.rjust(3,'0')
f=f.ljust(11,'0')
cap="e=%s.%s" % (w,f)
text=font.render(cap,True,(255,255,255))
visual.blit(text,(xOff,44))
pygame.display.flip()
tstClassErrorPath='20LSTMCell/tst_ClassAccu'
networkPath='20LSTMCell/TrainUntilConv.xml'
figPath='20LSTMCell/ErrorGraph'
#####################
#####################
print "Training Data Length: ", len(trndata)
print "Num of Training Seq: ", trndata.getNumSequences()
print "Validation Data Length: ", len(tstdata)
print "Num of Validation Seq: ", tstdata.getNumSequences()
print 'Start Training'
time_start = time.time()
while (tstErrorCount<100):
print "********** Classification with 20LSTMCell with RP- **********"
trnError=trainer.train()
tstError = trainer.testOnData(dataset=tstdata)
trnAccu = 100-percentError(trainer.testOnClassData(), trndata['class'])
tstAccu = 100-percentError(trainer.testOnClassData(dataset=tstdata), tstdata['class'])
trn_class_accu.append(trnAccu)
tst_class_accu.append(tstAccu)
trn_error.append(trnError)
tst_error.append(tstError)
np.savetxt(trnErrorPath, trn_error)
np.savetxt(tstErrorPath, tst_error)
np.savetxt(trnClassErrorPath, trn_class_accu)
np.savetxt(tstClassErrorPath, tst_class_accu)
if(oldtstError==0):
oldtstError = tstError
snd_20043 = load_snd(20043)
append2DS(DS, snd_20043, 2, nClasses)
# fnn = buildNetwork(1, 15, 5, hiddenclass = LSTMLayer, outclass = SoftmaxLayer, outputbias = False, recurrent = True)
fnn = buildNetwork(1, 1, nClasses, hiddenclass=LSTMLayer, outclass=TanhLayer, outputbias=False, recurrent=True)
# Create a trainer for backprop and train the net.
# trainer = BackpropTrainer(fnn, DStrain, learningrate = 0.005)
trainer = RPropMinusTrainer(fnn, dataset=DS, verbose=True)
for i in range(4):
# train the network for 1 epoch
trainer.trainEpochs(1)
print trainer.train()
fnn.reset()
summed = numpy.zeros(nClasses)
for sample in snd_18768:
summed += fnn.activate([sample])
print summed / len(snd_18768)
fnn.reset()
summed = numpy.zeros(nClasses)
for sample in snd_21649:
summed += fnn.activate([sample])
print summed / len(snd_21649)
fnn.reset()
summed = numpy.zeros(nClasses)
ds.addSample((arg), (r))
trainer_big = BackpropTrainer(net_big,
ds,
learningrate=0.01,
lrdecay=1.0,
momentum=0.0,
weightdecay=0.0)
# RProp-, cf. [Igel&Huesken, Neurocomputing 50, 2003
trainer = RPropMinusTrainer(net, dataset=ds)
# trainer.trainUntilConvergence()
for i in range(100):
trainer.train()
for i in range(10):
trainer_big.train()
for i in range(n):
arg = (i / n) * 6 - 3
y_n.append(net.activate([arg]))
y_n_big.append(net_big.activate([arg]))
fig = plt.figure()
plt.plot(x, y, 'black')
plt.plot(x, y_noise, 'r')
plt.plot(x, y_n, 'b')
plt.plot(x, y_n_big, 'g')
def recurrent_neural_network(self,
train_X,
train_y,
test_X,
test_y,
n_hidden_neurons=50,
iterations=100,
gridsearch=False,
gridsearch_training_frac=0.7,
outputbias=False,
error='accuracy'):
if gridsearch:
n_hidden_neurons, iterations, outputbias = self.gridsearch_recurrent_neural_network(
train_X,
train_y,
test_X,
test_y,
gridsearch_training_frac=gridsearch_training_frac,
error=error)
# Create numerical datasets first.
new_train_X, new_test_X = self.create_numerical_multiple_dataset(
train_X, test_X)
new_train_y, new_test_y = self.create_numerical_multiple_dataset(
train_y, test_y)
# We normalize the input.....
new_train_X, new_test_X, min_X, max_X = self.normalize(
new_train_X, new_test_X, 0, 1)
new_train_y, new_test_y, min_y, max_y = self.normalize(
new_train_y, new_test_y, 0.1, 0.9)
# Create the proper pybrain datasets.
ds_training = self.rnn_dataset(new_train_X, new_train_y)
ds_test = self.rnn_dataset(new_test_X, new_test_y)
inputs = len(new_train_X.columns)
outputs = len(new_train_y.columns)
# Build the network with the proper parameters.
n = buildNetwork(inputs,
n_hidden_neurons,
outputs,
hiddenclass=SigmoidLayer,
outclass=SigmoidLayer,
outputbias=outputbias,
recurrent=True)
# Train using back propagation through time.
#trainer = BackpropTrainer(n, dataset=ds_training, verbose=False, momentum=0.9, learningrate=0.01)
trainer = RPropMinusTrainer(n, dataset=ds_training, verbose=False)
for i in range(0, iterations):
trainer.train()
# for mod in n.modules:
# for conn in n.connections[mod]:
# print conn
# for cc in range(len(conn.params)):
# print conn.whichBuffers(cc), conn.params[cc]
# Determine performance on the training and test set....
# Y_train = []
# for i in range(0, len(new_train_X.index)):
# input = tuple(new_train_X.ix[i,:].values)
# output = n.activate(input)
# Y_train.append(output)
# Y_test = []
# for i in range(0, len(new_test_X.index)):
# Y_test.append(n.activate(tuple(new_test_X.ix[i,:].values)))
Y_train = []
Y_test = []
for sample, target in ds_training.getSequenceIterator(0):
Y_train.append(n.activate(sample).tolist())
for sample, target in ds_test.getSequenceIterator(0):
Y_test.append(n.activate(sample).tolist())
y_train_result = pd.DataFrame(Y_train,
columns=new_train_y.columns,
index=train_y.index)
y_test_result = pd.DataFrame(Y_test,
columns=new_test_y.columns,
index=test_y.index)
# print y_train_result
y_train_result = self.denormalize(y_train_result, min_y, max_y, 0.1,
0.9)
y_test_result = self.denormalize(y_test_result, min_y, max_y, 0.1, 0.9)
# plot.plot(train_y.index, train_y)
# plot.hold(True)
# plot.plot(train_y.index, pred_train_y_prob)
# plot.show()
return y_train_result.idxmax(axis=1), y_test_result.idxmax(
axis=1), y_train_result, y_test_result
visual.blit(text,(4,44))
n.reset()
prevCh=EOL
prevGuess=0.0
se=0
n.reset() # reset to t0 for training sequence
for i in range(len(target)):
curCh=ord(target[i])
tgtTicker[i]=chr(curCh)
inpTicker[i]=chr(max(1,prevCh))
ds.clear()
# scales bytes to range [0,1]
ds.addSample((prevCh/255.0),(curCh/255.0,))
#ds.addSample((prevCh/255.0,prevGuess),(curCh/255.0,))
err=trainer.train()
prevGuess=n['out'].outputbuffer[0][0]
outVh=min(255.0,max(0.0,255.0*prevGuess))
outCh=int(round(outVh))
outVh=outVh-outCh
if 32>outCh or 127<outCh:
outCh=1.0
outTicker[i]=chr(int(outCh))
outVTicker[i]=round(20.0*outVh)
se+=(255.0*err)**2
prevCh=curCh
mse=math.sqrt(se)
smse+=mse**2
if rendering:
frac=str(mse).split('.')
