def __init__(self, model_type='SparseFilter', weight_dims=(100, 256), layer_input=None, p=None, group_size=None, step=None, lr=0.01, c='n', weights=None): """ Builds a layer for the network by constructing a model. Parameters: ---------- model_type : str The model type to build into a given layer. weight_dims : list of tuples The dimensions of the weight matrices for each layer. fully connected: [neurons x input_dim ^ 2] convolutional: [filters x dim x dim] layer_input : ndarray (symbolic Theano variable) The input to a given layer. p : int The pooling size (assumed to be square). group_size : int The group size for group sparse filtering. step : int The step size for group sparse filtering. lr : int The learning rate for gradient descent. """ # assign network inputs to layer self.m = model_type self.weight_dims = weight_dims self.x = layer_input self.p = p self.lr = lr self.c = c if weights is None: self.w = init_weights(weight_dims) elif weights is not None: self.w = weights # build model based on model_type self.model = None if model_type == 'SparseFilter': self.model = SparseFilter(self.w, self.x) elif model_type == 'ConvolutionalSF': self.model = ConvolutionalSF(self.w, self.x) elif model_type == 'GroupSF': self.g_mat = connections.gMatToroidal(self.weight_dims[0], group_size, step, centered='n') self.model = GroupSF(self.w, self.x, self.g_mat) elif model_type == 'MultiGroupSF': self.g_mat1 = connections.groupMat(self.weight_dims[0], group_size, step) self.g_mat2 = connections.groupMat( np.square(np.sqrt(self.weight_dims[0]) / 2), group_size, step, ) self.model = MultiGroupSF(self.w, self.x, self.g_mat1, self.g_mat2) elif model_type == 'GroupConvolutionalSF': self.g_mat = connections.gMatToroidal(self.weight_dims[0], group_size, step, centered='n') self.model = GroupConvolutionalSF(self.w, self.x, self.g_mat) assert self.model is not None
def __init__(self, model_type='SparseFilter', weight_dims=(100, 256), layer_input=None, p=None, group_size=None, step=None, lr=0.01, c = 'n'): """ Builds a layer for the network by constructing a model. Parameters: ---------- model_type : str The model type to build into a given layer. weight_dims : list of tuples The dimensions of the weight matrices for each layer. fully connected: [neurons x input_dim ^ 2] convolutional: [filters x dim x dim] layer_input : ndarray (symbolic Theano variable) The input to a given layer. p : int The pooling size (assumed to be square). group_size : int The group size for group sparse filtering. step : int The step size for group sparse filtering. lr : int The learning rate for gradient descent. """ # assign network inputs to layer self.m = model_type self.weight_dims = weight_dims self.w = init_weights(weight_dims) # TODO: constrain L2-norm of weights to sum to unity self.x = layer_input self.p = p self.lr = lr self.c = c # build model based on model_type self.model = None if model_type == 'SparseFilter': self.model = SparseFilter(self.w, self.x) elif model_type == 'ConvolutionalSF': self.model = ConvolutionalSF(self.w, self.x) elif model_type == 'GroupSF': self.g_mat = connections.gMatToroidal(self.weight_dims[0], group_size, step, centered='n') self.model = GroupSF(self.w, self.x, self.g_mat) elif model_type == 'GroupConvolutionalSF': self.g_mat = connections.gMatToroidal(self.weight_dims[0], group_size, step, centered='n') self.model = GroupConvolutionalSF(self.w, self.x, self.g_mat) assert self.model is not None
def __init__(self, model_type='SparseFilter', weight_dims=(100, 256), layer_input=None, lr=0.0001, dim=2, weights=None, p=None): """ Builds a layer for the network by constructing a model. Parameters: ---------- model_type : str The model type to build into a given layer. weight_dims : list of tuples The dimensions of the weight matrices for each layer. fully connected: [neurons x input_dim ^ 2] convolutional: [filters x dim x dim] layer_input : ndarray (symbolic Theano variable) The input to a given layer. p : int The pooling size (assumed to be square). group_size : int The group size for group sparse filtering. step : int The step size for group sparse filtering. lr : int The learning rate for gradient descent. """ # assign network inputs to layer self.m = model_type self.weight_dims = weight_dims self.x = layer_input self.lr = lr self.dim = dim self.p = p if weights is None: self.w = init_weights(weight_dims) elif weights is not None: self.w = weights # build model based on model_type self.model = None if model_type == 'SparseFilter': self.model = SparseFilter(self.w, self.x) elif model_type == 'ConvolutionalSF': self.model = ConvolutionalSF(self.w, self.x) elif model_type == 'ConvolutionalSF3D': self.model = ConvolutionalSF3D(self.w, self.x, self.p) assert self.model is not None
def image_synthesis_function(self): image = init_weights((1, weights.shape[1])) synthesis_fns = [] for l in self.layers: # get f = l.get_activations() fn = theano.function( inputs=[], outputs=[f], givens={ self.x: image }, on_unused_input='ignore' ) synthesis_fns.append(fn) return synthesis_fns
print "compiling theano functions..." train, _, _ = model_.training_functions(data) # train the sparse filtering network print "training network..." for epoch in xrange(200): # 100 cost, weights = train[0](index=0) print("Layer %i cost at epoch %i and batch %i: %f" % (1, epoch, 0, cost)) elif convolutional == 'y': print "setting up network for" + model X = t.ftensor4() Y = t.fmatrix() w_out = init.init_weights((train_out[model].shape[1], 10)) py_x = final_layer(X, w_out) y_x = t.argmax(py_x, axis=1) cost = t.mean(t.nnet.categorical_crossentropy(py_x, Y)) params = [w_out] updates = RMSprop(cost, params, lr=0.001) print "compiling theano functions..." train = theano.function(inputs=[X, Y], outputs=[cost, w_out], updates=updates, allow_input_downcast=True) predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True) print "training fully connected layer..." max_iter = 100 batch_size = 1000
data = scaling.lcn_3d_input(data, kernel_shape=[3, 9, 9], n_maps=data.shape[2]) print 'done!' return data mri_data = data_read('data.mat') mri_data = shuffle_data(mri_data) mri_data = mri_data[0] mri_data = preprocess_data(mri_data.reshape(19, 30, 1, 256, 256)) print mri_data.shape ftensor5 = t.TensorType('float32', [False] * 5) X = ftensor5('x') w1 = init_weights([13, 3, 1, 11, 11]) # out after (1, 4, 4) max-pooling: [examples, 28, filters, 62, 62] w2 = init_weights([13, 3, 13, 6, 6]) # out after (2, 4, 4) max-pooling: [examples, 13, filters, 15, 15] w3 = init_weights([13, 3, 13, 4, 4]) # out after (4, 4, 4) max-pooling: [examples, 3, filters, 3, 3] w4 = init_weights([13, 13 * 3 * 3 * 3]) # out: [examples, filters] # w4 = init_weights([13, 2, 13, 3, 3]) # out after max-pooling: [examples, 1, filters, 13, 13] # w5 = init_weights([13, 13, 2, 2]) # out after (4, 4) max-pooling: [examples, filters, 3, 3] # w6 = init_weights([13, 13 * 3 * 3]) ####################### conv_out1 = conv3d( signals=X, filters=w1 )
neurons = 625 kernel_size = 11 examples = data.shape[0] if convolutional == 'y': out_dim = data.shape[3] - (kernel_size - 1) elif convolutional == 'n': input_dim = data.shape[1] batch_size = 1000 epochs = 100 n_batches = examples / batch_size # create symbolic variables for the model if convolutional == 'y': input_ = t.ftensor4() weights = init_weights((neurons, channels, kernel_size, kernel_size)) biases = theano.shared(np.asarray(np.zeros(neurons), dtype=theano.config.floatX)) inhibition = theano.shared(np.asarray(np.zeros(neurons), dtype=theano.config.floatX)) target = theano.shared(np.asarray(np.zeros((batch_size, neurons, out_dim, out_dim)), dtype=theano.config.floatX)) elif convolutional == 'n': input_ = t.fmatrix() weights = init_weights((neurons, input_dim)) biases = theano.shared(np.asarray(np.zeros(neurons), dtype=theano.config.floatX)) inhibition = theano.shared(np.asarray(np.zeros(neurons), dtype=theano.config.floatX)) target = theano.shared(np.asarray(np.zeros((batch_size, neurons)), dtype=theano.config.floatX)) print "building the model..." model = ELPS(weights, biases, input_, inhibition, target, neurons, examples, batch_size, convolutional) print "getting training functions..." train = model.training_function()
train, _, _ = model_.training_functions(data) # train the sparse filtering network print "training network..." for epoch in xrange(200): # 100 cost, weights = train[0](index=0) print("Layer %i cost at epoch %i and batch %i: %f" % (1, epoch, 0, cost)) elif convolutional == 'y': print "setting up network for" + model X = t.ftensor4() Y = t.fmatrix() w_out = init.init_weights((train_out[model].shape[1], 10)) py_x = final_layer(X, w_out) y_x = t.argmax(py_x, axis=1) cost = t.mean(t.nnet.categorical_crossentropy(py_x, Y)) params = [w_out] updates = RMSprop(cost, params, lr=0.001) print "compiling theano functions..." train = theano.function(inputs=[X, Y], outputs=[cost, w_out], updates=updates, allow_input_downcast=True) predict = theano.function(inputs=[X], outputs=y_x,
# define important variables neurons = weights.shape[0] dim = np.sqrt(weights.shape[1]) max_iter = 10000 # create data structures to save outputs images_saved = np.zeros(weights.shape) # loop over neurons for neuron in xrange(neurons): # # create symbolic variable for synthesized image # image = t.fvector('image') # create randomly initialized images with the same size as the neuron receptive fields image = init.init_weights((1, weights.shape[1])) # get the activation value of the neuron activation = t.dot(model.layers[0].get_weights()[neuron], image.T) # using raw activation value # _, _, _, _, activations, _ = outputs[model.n_layers - 1](image.get_value()) # activation = activations[neuron] # todo: this will not scale easily to deep architecture; should pass to Network # create cost function for evaluation Lambda = .1 # parameter of l2 shrinkage cost = -(activation[0] - Lambda * l2_norm(image)) # negative in front for gradient ascent # put parameters into list parameters = image # get updates