def __init__(self, model_type='SparseFilter', weight_dims=(100, 256), layer_input=None,
                 p=None, group_size=None, step=None, lr=0.01, c='n', weights=None):
        
        """
        Builds a layer for the network by constructing a model. 
        
        Parameters: 
        ----------
        model_type : str
            The model type to build into a given layer. 
        weight_dims : list of tuples
            The dimensions of the weight matrices for each layer. 
            fully connected: [neurons x input_dim ^ 2]
            convolutional: [filters x dim x dim]
        layer_input : ndarray (symbolic Theano variable)
            The input to a given layer. 
        p : int
            The pooling size (assumed to be square).
        group_size : int
            The group size for group sparse filtering. 
        step : int
            The step size for group sparse filtering. 
        lr : int
            The learning rate for gradient descent. 
        """
        
        # assign network inputs to layer
        self.m = model_type
        self.weight_dims = weight_dims
        self.x = layer_input
        self.p = p    
        self.lr = lr
        self.c = c

        if weights is None:
            self.w = init_weights(weight_dims)
        elif weights is not None:
            self.w = weights
        
        # build model based on model_type
        self.model = None
        if model_type == 'SparseFilter':
            self.model = SparseFilter(self.w, self.x)
        elif model_type == 'ConvolutionalSF':
            self.model = ConvolutionalSF(self.w, self.x)
        elif model_type == 'GroupSF':
            self.g_mat = connections.gMatToroidal(self.weight_dims[0], group_size, step, centered='n')
            self.model = GroupSF(self.w, self.x, self.g_mat)
        elif model_type == 'MultiGroupSF':
            self.g_mat1 = connections.groupMat(self.weight_dims[0], group_size, step)
            self.g_mat2 = connections.groupMat(
                np.square(np.sqrt(self.weight_dims[0]) / 2),
                group_size, step,
            )
            self.model = MultiGroupSF(self.w, self.x, self.g_mat1, self.g_mat2)
        elif model_type == 'GroupConvolutionalSF':
            self.g_mat = connections.gMatToroidal(self.weight_dims[0], group_size, step, centered='n')
            self.model = GroupConvolutionalSF(self.w, self.x, self.g_mat)
        assert self.model is not None
 def __init__(self, model_type='SparseFilter', weight_dims=(100, 256), layer_input=None,
              p=None, group_size=None, step=None, lr=0.01, c = 'n'):   
     
     """
     Builds a layer for the network by constructing a model. 
     
     Parameters: 
     ----------
     model_type : str
         The model type to build into a given layer. 
     weight_dims : list of tuples
         The dimensions of the weight matrices for each layer. 
         fully connected: [neurons x input_dim ^ 2]
         convolutional: [filters x dim x dim]
     layer_input : ndarray (symbolic Theano variable)
         The input to a given layer. 
     p : int
         The pooling size (assumed to be square).
     group_size : int
         The group size for group sparse filtering. 
     step : int
         The step size for group sparse filtering. 
     lr : int
         The learning rate for gradient descent. 
     """
     
     # assign network inputs to layer
     self.m = model_type
     self.weight_dims = weight_dims
     self.w = init_weights(weight_dims)  # TODO: constrain L2-norm of weights to sum to unity
     self.x = layer_input
     self.p = p    
     self.lr = lr
     self.c = c
     
     # build model based on model_type
     self.model = None
     if model_type == 'SparseFilter':
         self.model = SparseFilter(self.w, self.x)
     elif model_type == 'ConvolutionalSF':
         self.model = ConvolutionalSF(self.w, self.x)
     elif model_type == 'GroupSF':
         self.g_mat = connections.gMatToroidal(self.weight_dims[0], group_size, step, centered='n')
         self.model = GroupSF(self.w, self.x, self.g_mat)
     elif model_type == 'GroupConvolutionalSF':
         self.g_mat = connections.gMatToroidal(self.weight_dims[0], group_size, step, centered='n')
         self.model = GroupConvolutionalSF(self.w, self.x, self.g_mat)
     assert self.model is not None
Beispiel #3
0
    def __init__(self, model_type='SparseFilter', weight_dims=(100, 256), layer_input=None,
                 lr=0.0001, dim=2, weights=None, p=None):

        """
        Builds a layer for the network by constructing a model.

        Parameters:
        ----------
        model_type : str
            The model type to build into a given layer.
        weight_dims : list of tuples
            The dimensions of the weight matrices for each layer.
            fully connected: [neurons x input_dim ^ 2]
            convolutional: [filters x dim x dim]
        layer_input : ndarray (symbolic Theano variable)
            The input to a given layer.
        p : int
            The pooling size (assumed to be square).
        group_size : int
            The group size for group sparse filtering.
        step : int
            The step size for group sparse filtering.
        lr : int
            The learning rate for gradient descent.
        """

        # assign network inputs to layer
        self.m = model_type
        self.weight_dims = weight_dims
        self.x = layer_input
        self.lr = lr
        self.dim = dim
        self.p = p

        if weights is None:
            self.w = init_weights(weight_dims)
        elif weights is not None:
            self.w = weights

        # build model based on model_type
        self.model = None
        if model_type == 'SparseFilter':
            self.model = SparseFilter(self.w, self.x)
        elif model_type == 'ConvolutionalSF':
            self.model = ConvolutionalSF(self.w, self.x)
        elif model_type == 'ConvolutionalSF3D':
            self.model = ConvolutionalSF3D(self.w, self.x, self.p)
        assert self.model is not None
    def image_synthesis_function(self):

        image = init_weights((1, weights.shape[1]))

        synthesis_fns = []

        for l in self.layers:

            # get
            f = l.get_activations()

            fn = theano.function(
                inputs=[],
                outputs=[f],
                givens={
                    self.x: image
                },
                on_unused_input='ignore'
            )

            synthesis_fns.append(fn)

        return synthesis_fns
        print "compiling theano functions..."
        train, _, _ = model_.training_functions(data)

        # train the sparse filtering network
        print "training network..."
        for epoch in xrange(200):  # 100

            cost, weights = train[0](index=0)
            print("Layer %i cost at epoch %i and batch %i: %f" % (1, epoch, 0, cost))

    elif convolutional == 'y':

        print "setting up network for" + model
        X = t.ftensor4()
        Y = t.fmatrix()
        w_out = init.init_weights((train_out[model].shape[1], 10))

        py_x = final_layer(X, w_out)
        y_x = t.argmax(py_x, axis=1)

        cost = t.mean(t.nnet.categorical_crossentropy(py_x, Y))
        params = [w_out]
        updates = RMSprop(cost, params, lr=0.001)

        print "compiling theano functions..."
        train = theano.function(inputs=[X, Y], outputs=[cost, w_out], updates=updates, allow_input_downcast=True)
        predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)

        print "training fully connected layer..."
        max_iter = 100
        batch_size = 1000
Beispiel #6
0
    data = scaling.lcn_3d_input(data, kernel_shape=[3, 9, 9], n_maps=data.shape[2])
    print 'done!'

    return data


mri_data = data_read('data.mat')
mri_data = shuffle_data(mri_data)
mri_data = mri_data[0]
mri_data = preprocess_data(mri_data.reshape(19, 30, 1, 256, 256))
print mri_data.shape

ftensor5 = t.TensorType('float32', [False] * 5)

X = ftensor5('x')
w1 = init_weights([13, 3, 1, 11, 11])    # out after (1, 4, 4) max-pooling: [examples, 28, filters, 62, 62]
w2 = init_weights([13, 3, 13, 6, 6])     # out after (2, 4, 4) max-pooling: [examples, 13, filters, 15, 15]
w3 = init_weights([13, 3, 13, 4, 4])     # out after (4, 4, 4) max-pooling: [examples, 3, filters, 3, 3]

w4 = init_weights([13, 13 * 3 * 3 * 3])       # out: [examples, filters]

# w4 = init_weights([13, 2, 13, 3, 3])        # out after max-pooling: [examples, 1, filters, 13, 13]
# w5 = init_weights([13, 13, 2, 2])           # out after (4, 4) max-pooling: [examples, filters, 3, 3]
# w6 = init_weights([13, 13 * 3 * 3])


#######################
conv_out1 = conv3d(
    signals=X,
    filters=w1
)
Beispiel #7
0
neurons = 625
kernel_size = 11
examples = data.shape[0]
if convolutional == 'y':
    out_dim = data.shape[3] - (kernel_size - 1)
elif convolutional == 'n':
    input_dim = data.shape[1]

batch_size = 1000
epochs = 100
n_batches = examples / batch_size

# create symbolic variables for the model
if convolutional == 'y':
    input_ = t.ftensor4()
    weights = init_weights((neurons, channels, kernel_size, kernel_size))
    biases = theano.shared(np.asarray(np.zeros(neurons), dtype=theano.config.floatX))
    inhibition = theano.shared(np.asarray(np.zeros(neurons), dtype=theano.config.floatX))
    target = theano.shared(np.asarray(np.zeros((batch_size, neurons, out_dim, out_dim)), dtype=theano.config.floatX))
elif convolutional == 'n':
    input_ = t.fmatrix()
    weights = init_weights((neurons, input_dim))
    biases = theano.shared(np.asarray(np.zeros(neurons), dtype=theano.config.floatX))
    inhibition = theano.shared(np.asarray(np.zeros(neurons), dtype=theano.config.floatX))
    target = theano.shared(np.asarray(np.zeros((batch_size, neurons)), dtype=theano.config.floatX))

print "building the model..."
model = ELPS(weights, biases, input_, inhibition, target, neurons, examples, batch_size, convolutional)

print "getting training functions..."
train = model.training_function()
Beispiel #8
0
        train, _, _ = model_.training_functions(data)

        # train the sparse filtering network
        print "training network..."
        for epoch in xrange(200):  # 100

            cost, weights = train[0](index=0)
            print("Layer %i cost at epoch %i and batch %i: %f" %
                  (1, epoch, 0, cost))

    elif convolutional == 'y':

        print "setting up network for" + model
        X = t.ftensor4()
        Y = t.fmatrix()
        w_out = init.init_weights((train_out[model].shape[1], 10))

        py_x = final_layer(X, w_out)
        y_x = t.argmax(py_x, axis=1)

        cost = t.mean(t.nnet.categorical_crossentropy(py_x, Y))
        params = [w_out]
        updates = RMSprop(cost, params, lr=0.001)

        print "compiling theano functions..."
        train = theano.function(inputs=[X, Y],
                                outputs=[cost, w_out],
                                updates=updates,
                                allow_input_downcast=True)
        predict = theano.function(inputs=[X],
                                  outputs=y_x,
# define important variables
neurons = weights.shape[0]
dim = np.sqrt(weights.shape[1])
max_iter = 10000

# create data structures to save outputs
images_saved = np.zeros(weights.shape)

# loop over neurons
for neuron in xrange(neurons):

    # # create symbolic variable for synthesized image
    # image = t.fvector('image')

    # create randomly initialized images with the same size as the neuron receptive fields
    image = init.init_weights((1, weights.shape[1]))

    # get the activation value of the neuron
    activation = t.dot(model.layers[0].get_weights()[neuron], image.T)  # using raw activation value
    # _, _, _, _, activations, _ = outputs[model.n_layers - 1](image.get_value())
    # activation = activations[neuron]
    # todo: this will not scale easily to deep architecture; should pass to Network

    # create cost function for evaluation
    Lambda = .1  # parameter of l2 shrinkage
    cost = -(activation[0] - Lambda * l2_norm(image))  # negative in front for gradient ascent

    # put parameters into list
    parameters = image

    # get updates