def learn_curve(network, training_input, training_target, validation_input, validation_target, epochs=50):
'''Plots the erros w.r.t the number of epochs of training. Assumes no regularisation.
Returns two lists containing the training and validation errors.'''
training_errors = []
validation_errors = []
num_training = len(training_input)
#Train mini batches of 10s
minibatches = [(training_input[i:i+10, :], training_target[i:i+10, :]) for i in range(0, num_training, 10)]
for epoch in range(epochs):
for batch in minibatches:
backprop.train(network, batch[0], batch[1])
#Get errors for this epoch
train_error = error(network, training_input, training_target)
val_error = error(network, validation_input, validation_target)
training_errors.append(train_error)
validation_errors.append(val_error)
#Plot the data
pylab.plot(training_errors, label="Training Error")
pylab.plot(validation_errors, label="Validation Error")
pylab.title("Training and Validation Errors w.r.t. Number of Epochs")
pylab.xlabel("Epochs")
pylab.ylabel("Error")
pylab.legend()
pylab.show()
return training_errors, validation_errors
def train_ae(ae,
training_inputs,
epochs=100,
learning_rate=0.2,
batch_size=100,
noise=0.4,
tau=0,
get_error=False):
'''Trains an autoencoder on data.'''
errors = []
for epoch in range(epochs):
corrupt_training_inputs = dropout(
training_inputs, noise) #Add noise to input if you want
input_batches = [
training_inputs[i:i + batch_size]
for i in range(0, training_inputs.shape[0], batch_size)
] #Split to mini-batches
corrupt_input_batches = [
corrupt_training_inputs[i:i + batch_size]
for i in range(0, training_inputs.shape[0], batch_size)
]
print 'Epoch Number: ', epoch
for i in range(len(input_batches)):
train(ae, corrupt_input_batches[i], input_batches[i],
learning_rate, tau)
if i % 100 == 0 and get_error:
e = error(ae, input_batches[i], input_batches[i], tau)
print e
errors.append(e)
if get_error:
return errors
def vary_data(network, training_input, training_target, validation_input, validation_target, epochs=5):
'''Plots training and validation errors of a net w.r.t. the number of training
cases used in learning. Useful for determining if the issue is in the amount of data
Returns two lists containing the training and validation errors'''
training_errors = []
validation_errors = []
num_training = len(training_input)
for percent in range(5, 105, 5): #Sweep through percentages from 5 to 100%
train_length = num_training*percent/100.0
validation_length = len(validation_input)*percent/100.0
network_copy = network
#Train mini batches of 10s
minibatches = [(training_input[i:i+10, :], training_target[i:i+10, :]) for i in range(0, int(train_length), 10)]
for epoch in range(epochs):
for batch in minibatches:
backprop.train(network_copy, batch[0], batch[1])
#Get errors for this percent of data
train_error = error(network_copy, training_input[0:train_length], training_target[0:train_length])
val_error = error(network_copy, validation_input[0:validation_length], validation_target[0:validation_length])
training_errors.append(train_error)
validation_errors.append(val_error)
#Plot the data
pylab.plot(range(5,105,5), training_errors, label="Training Error")
pylab.plot(range(5,105,5), validation_errors, label="Validation Error")
pylab.title("Training and Validation Errors w.r.t. % of Data used")
pylab.xlabel("Data Used (%)")
pylab.ylabel("Error After "+ str(epochs)+ " Epochs")
pylab.legend()
pylab.show()
return training_errors, validation_errors
def reg_parameter(network,
training_input,
training_target,
validation_input,
validation_target,
sweep_range=(0.01, 20),
epochs=10):
'''Plots training and validation errors of a net varying the regularisation parameter over a set range. Useful for determining
where network is underfitting and overfitting. Sweep Range is a tuple, where the first value is the initial decay rate to test,
and the second value is the number of parameters to test. Each parameter will be 1.2 times the one before it.
Returns two lists containing the training and validation errors'''
training_errors = []
validation_errors = []
num_training = len(training_input)
start_rate = sweep_range[0]
num_parameters = sweep_range[1]
decay_rates = [
start_rate * (1.2**i) for i in range(num_parameters)
] #Create a list of different regularisation parameters to try.
decay_rates.insert(0, 0)
for rate in decay_rates:
#Train mini batches of 10s
new_net = network
minibatches = [(training_input[i:i + 10, :],
training_target[i:i + 10, :])
for i in range(0, num_training, 10)]
for epoch in range(epochs):
for batch in minibatches:
backprop.train(new_net, batch[0], batch[1], rate)
#Get errors for this decay rate
train_error = error(new_net, training_input, training_target, rate)
val_error = error(new_net, validation_input, validation_target, rate)
training_errors.append(train_error)
validation_errors.append(val_error)
#Plot the data
pylab.plot(decay_rates, training_errors, label="Training Error")
pylab.plot(decay_rates, validation_errors, label="Validation Error")
pylab.title(
"Training and Validation Errors w.r.t. Regularisation Parameter")
pylab.xlabel("Regularisation Parameter")
pylab.ylabel("Error After 10 Epochs")
pylab.legend()
pylab.show()
return training_errors, validation_errors
def train_ae(ae, training_inputs, epochs=100, learning_rate=0.2, batch_size=100, noise=0.4, tau=0, get_error=False):
'''Trains an autoencoder on data.'''
errors = []
for epoch in range(epochs):
corrupt_training_inputs = dropout(training_inputs, noise) #Add noise to input if you want
input_batches = [training_inputs[i:i+batch_size] for i in range(0, training_inputs.shape[0], batch_size)] #Split to mini-batches
corrupt_input_batches = [corrupt_training_inputs[i:i+batch_size] for i in range(0, training_inputs.shape[0], batch_size)]
print 'Epoch Number: ', epoch
for i in range(len(input_batches)):
train(ae, corrupt_input_batches[i], input_batches[i], learning_rate, tau)
if i%100 == 0 and get_error:
e = error(ae, input_batches[i], input_batches[i], tau)
print e
errors.append(e)
if get_error:
return errors
def vary_data(network,
training_input,
training_target,
validation_input,
validation_target,
epochs=5):
'''Plots training and validation errors of a net w.r.t. the number of training
cases used in learning. Useful for determining if the issue is in the amount of data
Returns two lists containing the training and validation errors'''
training_errors = []
validation_errors = []
num_training = len(training_input)
for percent in range(5, 105, 5): #Sweep through percentages from 5 to 100%
train_length = num_training * percent / 100.0
validation_length = len(validation_input) * percent / 100.0
network_copy = network
#Train mini batches of 10s
minibatches = [(training_input[i:i + 10, :],
training_target[i:i + 10, :])
for i in range(0, int(train_length), 10)]
for epoch in range(epochs):
for batch in minibatches:
backprop.train(network_copy, batch[0], batch[1])
#Get errors for this percent of data
train_error = error(network_copy, training_input[0:train_length],
training_target[0:train_length])
val_error = error(network_copy, validation_input[0:validation_length],
validation_target[0:validation_length])
training_errors.append(train_error)
validation_errors.append(val_error)
#Plot the data
pylab.plot(range(5, 105, 5), training_errors, label="Training Error")
pylab.plot(range(5, 105, 5), validation_errors, label="Validation Error")
pylab.title("Training and Validation Errors w.r.t. % of Data used")
pylab.xlabel("Data Used (%)")
pylab.ylabel("Error After " + str(epochs) + " Epochs")
pylab.legend()
pylab.show()
return training_errors, validation_errors
def reg_parameter(network, training_input, training_target, validation_input, validation_target, sweep_range=(0.01, 20), epochs=10):
'''Plots training and validation errors of a net varying the regularisation parameter over a set range. Useful for determining
where network is underfitting and overfitting. Sweep Range is a tuple, where the first value is the initial decay rate to test,
and the second value is the number of parameters to test. Each parameter will be 1.2 times the one before it.
Returns two lists containing the training and validation errors'''
training_errors = []
validation_errors = []
num_training = len(training_input)
start_rate = sweep_range[0]
num_parameters = sweep_range[1]
decay_rates = [start_rate*(1.2**i) for i in range(num_parameters)] #Create a list of different regularisation parameters to try.
decay_rates.insert(0,0)
for rate in decay_rates:
#Train mini batches of 10s
new_net = network
minibatches = [(training_input[i:i+10, :], training_target[i:i+10, :]) for i in range(0, num_training, 10)]
for epoch in range(epochs):
for batch in minibatches:
backprop.train(new_net, batch[0], batch[1], rate)
#Get errors for this decay rate
train_error = error(new_net, training_input, training_target, rate)
val_error = error(new_net, validation_input, validation_target, rate)
training_errors.append(train_error)
validation_errors.append(val_error)
#Plot the data
pylab.plot(decay_rates, training_errors, label="Training Error")
pylab.plot(decay_rates,validation_errors, label="Validation Error")
pylab.title("Training and Validation Errors w.r.t. Regularisation Parameter")
pylab.xlabel("Regularisation Parameter")
pylab.ylabel("Error After 10 Epochs")
pylab.legend()
pylab.show()
return training_errors, validation_errors
def learn_curve(network,
training_input,
training_target,
validation_input,
validation_target,
epochs=50):
'''Plots the erros w.r.t the number of epochs of training. Assumes no regularisation.
Returns two lists containing the training and validation errors.'''
training_errors = []
validation_errors = []
num_training = len(training_input)
#Train mini batches of 10s
minibatches = [(training_input[i:i + 10, :], training_target[i:i + 10, :])
for i in range(0, num_training, 10)]
for epoch in range(epochs):
for batch in minibatches:
backprop.train(network, batch[0], batch[1])
#Get errors for this epoch
train_error = error(network, training_input, training_target)
val_error = error(network, validation_input, validation_target)
training_errors.append(train_error)
validation_errors.append(val_error)
#Plot the data
pylab.plot(training_errors, label="Training Error")
pylab.plot(validation_errors, label="Validation Error")
pylab.title("Training and Validation Errors w.r.t. Number of Epochs")
pylab.xlabel("Epochs")
pylab.ylabel("Error")
pylab.legend()
pylab.show()
return training_errors, validation_errors
from sys import argv
from backprop import train
from backprop import test
if __name__ == "__main__":
if len(argv) == 1:
pass
elif argv[1] == 'train':
train()
elif argv[1] == 'test':
test()
else:
pass
