def test(self, input_matrix, teacher_matrix): """Function to test the network @param input_matrix -- matrix consisting of input data to the network. @param teacher_matrix -- matrix consisting of labels of input data . """ number_of_pictures = input_matrix.shape[-1] mse = 0 squared_errors = cp.dev_matrix_cmf(self.neuron_layer[-1].deltas.h, self.neuron_layer[-1].deltas.w) for batch in xrange(number_of_pictures/self.batch_size): index_begin = self.batch_size * batch index_end = index_begin + self.batch_size self.neuron_layer[0].activations = cp.push( input_matrix[:, index_begin:index_end].astype('float32').copy('F')) teachbatch = cp.push(teacher_matrix[:, index_begin:index_end].astype('float32').copy('F')) for i in xrange(self.number_of_layers): self.weight_layer[i].forward() cp.apply_binary_functor(squared_errors, self.neuron_layer[-1].deltas, cp.binary_functor.COPY) cp.apply_scalar_functor(squared_errors, cp.scalar_functor.SQUARE) mse += cp.sum(squared_errors) teachbatch.dealloc() print "MSE: ", (mse/number_of_pictures) squared_errors.dealloc()
def fit(self, input_matrix, teacher_matrix, n_epochs=100, learnrate = 0.10): """ Function to train the network @param input_matrix -- matrix consisting of input data to the network. @param teacher_matrix -- matrix consisting of labels of input data. @param n_epochs -- number of epochs the network is to be trained. """ n_samples = input_matrix.shape[-1] squared_errors = cp.dev_tensor_float_cm(self.neuron_layers[-1].deltas.shape) for r in xrange(n_epochs): print "Epoch ", r + 1, "/", n_epochs mse = 0.0 ce = 0.0 for batch in xrange(n_samples / self.batch_size): index_begin = self.batch_size * batch index_end = self.batch_size + index_begin # Push input and teacher to GPU memory # .copy("F") is needed since memory is non-contiguous self.neuron_layers[0].activations = cp.dev_tensor_float_cm( input_matrix[:, index_begin:index_end].copy('F')) teacher_batch_host = teacher_matrix[:, index_begin:index_end] teacher_batch = cp.dev_tensor_float_cm(teacher_batch_host.copy('F')) # Forward-Pass for i in xrange(self.n_layers): self.weight_layers[i].forward() # calculate error at output layer cp.copy(self.neuron_layers[-1].deltas, teacher_batch) self.neuron_layers[-1].deltas -= self.neuron_layers[-1].activations cp.copy(squared_errors, self.neuron_layers[-1].deltas) cp.apply_scalar_functor(squared_errors, cp.scalar_functor.SQUARE) mse += cp.sum(squared_errors) ce += float(np.sum(np.argmax(teacher_batch_host, axis=0) != np.argmax(self.neuron_layers[-1].activations.np, axis=0))) # Backward-Pass for i in xrange(self.n_layers): self.weight_layers[self.n_layers - i - 1].backward(learnrate, decay=.01) # Don't wait for garbage collector teacher_batch.dealloc() self.neuron_layers[0].activations.dealloc() print "MSE: ", (mse / n_samples) print "Classification Error Training: ", (ce / n_samples) squared_errors.dealloc()
def backward(self, learnrate=0.01, decay=0.0): """Backward pass, calculates the deltas of lower layer and updates the weights. @param learnrate how strongly the gradient influences the weights @param decay large values result in a regularization with to the squared weight value""" cp.prod(self.source.deltas, self.weight, self.target.deltas, 't', 'n') h = self.source.activations.copy() self.source.d_nonlinearity(h) self.source.deltas *= h h.dealloc() batch_size = self.source.activations.shape[1] dw = cp.prod(self.target.deltas, self.source.activations, 'n', 't') cp.learn_step_weight_decay(self.weight, dw, learnrate / batch_size, decay) dw.dealloc() db = cp.sum(self.target.deltas, 1) cp.learn_step_weight_decay(self.bias, db, learnrate / batch_size, decay) db.dealloc()
import cuv_python as cp import numpy as np h = np.zeros((1,256)) # create numpy matrix d = cp.dev_tensor_float(h) # constructs by copying numpy_array h2 = np.zeros((1,256)).copy("F") # create numpy matrix d2 = cp.dev_tensor_float_cm(h2) # creates dev_tensor_float_cm (column-major float) object cp.fill(d,1) # terse form cp.apply_nullary_functor(d,cp.nullary_functor.FILL,1) # verbose form h = d.np # pull and convert to numpy assert(np.sum(h) == 256) assert(cp.sum(d) == 256) d.dealloc() # explicitly deallocate memory (optional)
class MLP: """ A Multi-Layer Perceptron """ def __init__(self, neurons, batch_size): """Constructor @param neurons -- array of sizes of layers. @param batch_size -- size of batch being used for training. """ self.number_of_layers = len(neurons) - 1 self.batch_size = batch_size self.neuron_layer = [] self.weight_layer = [] for i in xrange(self.number_of_layers+1): dim1 = neurons[i] self.neuron_layer.append(neuron_layer(dim1, self.batch_size )) for i in xrange(self.number_of_layers): self.weight_layer.append(weight_layer(self.neuron_layer[i], self.neuron_layer[i+1])) def train(self, input_matrix, teacher_matrix, number_of_epochs): """Function to train the network @param input_matrix -- matrix consisting of input data to the network. @param teacher_matrix -- matrix consisting of labels of input data. @param number_of_epochs -- number of rounds the network is to be trained. """ number_of_pictures = input_matrix.shape[-1] squared_errors = cp.dev_matrix_cmf(self.neuron_layer[-1].deltas.h, self.neuron_layer[-1].deltas.w) for r in xrange(number_of_epochs): print "Epoch ", r+1, "/", number_of_epochs mse = 0 for batch in xrange(number_of_pictures/self.batch_size): index_begin = self.batch_size * batch index_end = self.batch_size + index_begin # Push input and teacher to GPU memory self.neuron_layer[0].activations = cp.push( input_matrix[:,index_begin:index_end].astype('float32').copy('F')) teachbatch = cp.push( teacher_matrix[:,index_begin:index_end].astype('float32').copy('F')) # Forward-Pass for i in xrange(self.number_of_layers): self.weight_layer[i].forward() # calculate error at output layer cp.apply_binary_functor(self.neuron_layer[-1].deltas, teachbatch, cp.binary_functor.COPY) cp.apply_binary_functor(self.neuron_layer[-1].deltas, self.neuron_layer[-1].activations, cp.binary_functor.SUBTRACT) cp.apply_binary_functor(squared_errors, self.neuron_layer[-1].deltas, cp.binary_functor.COPY) cp.apply_scalar_functor(squared_errors, cp.scalar_functor.SQUARE) mse += cp.sum(squared_errors) # Backward-Pass for i in xrange(self.number_of_layers): self.weight_layer[self.number_of_layers-i-1].backward() # Don't wait for garbage collector teachbatch.dealloc() self.neuron_layer[0].activations.dealloc() print "MSE: ", (mse/number_of_pictures) squared_errors.dealloc()