def __init__(self, model_type='SparseFilter', weight_dims=(100, 256), layer_input=None,
p=None, group_size=None, step=None, lr=0.01, c='n', weights=None):
"""
Builds a layer for the network by constructing a model.
Parameters:
----------
model_type : str
The model type to build into a given layer.
weight_dims : list of tuples
The dimensions of the weight matrices for each layer.
fully connected: [neurons x input_dim ^ 2]
convolutional: [filters x dim x dim]
layer_input : ndarray (symbolic Theano variable)
The input to a given layer.
p : int
The pooling size (assumed to be square).
group_size : int
The group size for group sparse filtering.
step : int
The step size for group sparse filtering.
lr : int
The learning rate for gradient descent.
"""
# assign network inputs to layer
self.m = model_type
self.weight_dims = weight_dims
self.x = layer_input
self.p = p
self.lr = lr
self.c = c
if weights is None:
self.w = init_weights(weight_dims)
elif weights is not None:
self.w = weights
# build model based on model_type
self.model = None
if model_type == 'SparseFilter':
self.model = SparseFilter(self.w, self.x)
elif model_type == 'ConvolutionalSF':
self.model = ConvolutionalSF(self.w, self.x)
elif model_type == 'GroupSF':
self.g_mat = connections.gMatToroidal(self.weight_dims[0], group_size, step, centered='n')
self.model = GroupSF(self.w, self.x, self.g_mat)
elif model_type == 'MultiGroupSF':
self.g_mat1 = connections.groupMat(self.weight_dims[0], group_size, step)
self.g_mat2 = connections.groupMat(
np.square(np.sqrt(self.weight_dims[0]) / 2),
group_size, step,
)
self.model = MultiGroupSF(self.w, self.x, self.g_mat1, self.g_mat2)
elif model_type == 'GroupConvolutionalSF':
self.g_mat = connections.gMatToroidal(self.weight_dims[0], group_size, step, centered='n')
self.model = GroupConvolutionalSF(self.w, self.x, self.g_mat)
assert self.model is not None
def __init__(self, model_type='SparseFilter', weight_dims=(100, 256), layer_input=None,
p=None, group_size=None, step=None, lr=0.01, c = 'n'):
"""
Builds a layer for the network by constructing a model.
Parameters:
----------
model_type : str
The model type to build into a given layer.
weight_dims : list of tuples
The dimensions of the weight matrices for each layer.
fully connected: [neurons x input_dim ^ 2]
convolutional: [filters x dim x dim]
layer_input : ndarray (symbolic Theano variable)
The input to a given layer.
p : int
The pooling size (assumed to be square).
group_size : int
The group size for group sparse filtering.
step : int
The step size for group sparse filtering.
lr : int
The learning rate for gradient descent.
"""
# assign network inputs to layer
self.m = model_type
self.weight_dims = weight_dims
self.w = init_weights(weight_dims) # TODO: constrain L2-norm of weights to sum to unity
self.x = layer_input
self.p = p
self.lr = lr
self.c = c
# build model based on model_type
self.model = None
if model_type == 'SparseFilter':
self.model = SparseFilter(self.w, self.x)
elif model_type == 'ConvolutionalSF':
self.model = ConvolutionalSF(self.w, self.x)
elif model_type == 'GroupSF':
self.g_mat = connections.gMatToroidal(self.weight_dims[0], group_size, step, centered='n')
self.model = GroupSF(self.w, self.x, self.g_mat)
elif model_type == 'GroupConvolutionalSF':
self.g_mat = connections.gMatToroidal(self.weight_dims[0], group_size, step, centered='n')
self.model = GroupConvolutionalSF(self.w, self.x, self.g_mat)
assert self.model is not None
def __init__(self, model_type='SparseFilter', weight_dims=(100, 256), layer_input=None,
lr=0.0001, dim=2, weights=None, p=None):
"""
Builds a layer for the network by constructing a model.
Parameters:
----------
model_type : str
The model type to build into a given layer.
weight_dims : list of tuples
The dimensions of the weight matrices for each layer.
fully connected: [neurons x input_dim ^ 2]
convolutional: [filters x dim x dim]
layer_input : ndarray (symbolic Theano variable)
The input to a given layer.
p : int
The pooling size (assumed to be square).
group_size : int
The group size for group sparse filtering.
step : int
The step size for group sparse filtering.
lr : int
The learning rate for gradient descent.
"""
# assign network inputs to layer
self.m = model_type
self.weight_dims = weight_dims
self.x = layer_input
self.lr = lr
self.dim = dim
self.p = p
if weights is None:
self.w = init_weights(weight_dims)
elif weights is not None:
self.w = weights
# build model based on model_type
self.model = None
if model_type == 'SparseFilter':
self.model = SparseFilter(self.w, self.x)
elif model_type == 'ConvolutionalSF':
self.model = ConvolutionalSF(self.w, self.x)
elif model_type == 'ConvolutionalSF3D':
self.model = ConvolutionalSF3D(self.w, self.x, self.p)
assert self.model is not None
def image_synthesis_function(self):
image = init_weights((1, weights.shape[1]))
synthesis_fns = []
for l in self.layers:
# get
f = l.get_activations()
fn = theano.function(
inputs=[],
outputs=[f],
givens={
self.x: image
},
on_unused_input='ignore'
)
synthesis_fns.append(fn)
return synthesis_fns
print "compiling theano functions..."
train, _, _ = model_.training_functions(data)
# train the sparse filtering network
print "training network..."
for epoch in xrange(200): # 100
cost, weights = train[0](index=0)
print("Layer %i cost at epoch %i and batch %i: %f" % (1, epoch, 0, cost))
elif convolutional == 'y':
print "setting up network for" + model
X = t.ftensor4()
Y = t.fmatrix()
w_out = init.init_weights((train_out[model].shape[1], 10))
py_x = final_layer(X, w_out)
y_x = t.argmax(py_x, axis=1)
cost = t.mean(t.nnet.categorical_crossentropy(py_x, Y))
params = [w_out]
updates = RMSprop(cost, params, lr=0.001)
print "compiling theano functions..."
train = theano.function(inputs=[X, Y], outputs=[cost, w_out], updates=updates, allow_input_downcast=True)
predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)
print "training fully connected layer..."
max_iter = 100
batch_size = 1000
data = scaling.lcn_3d_input(data, kernel_shape=[3, 9, 9], n_maps=data.shape[2])
print 'done!'
return data
mri_data = data_read('data.mat')
mri_data = shuffle_data(mri_data)
mri_data = mri_data[0]
mri_data = preprocess_data(mri_data.reshape(19, 30, 1, 256, 256))
print mri_data.shape
ftensor5 = t.TensorType('float32', [False] * 5)
X = ftensor5('x')
w1 = init_weights([13, 3, 1, 11, 11]) # out after (1, 4, 4) max-pooling: [examples, 28, filters, 62, 62]
w2 = init_weights([13, 3, 13, 6, 6]) # out after (2, 4, 4) max-pooling: [examples, 13, filters, 15, 15]
w3 = init_weights([13, 3, 13, 4, 4]) # out after (4, 4, 4) max-pooling: [examples, 3, filters, 3, 3]
w4 = init_weights([13, 13 * 3 * 3 * 3]) # out: [examples, filters]
# w4 = init_weights([13, 2, 13, 3, 3]) # out after max-pooling: [examples, 1, filters, 13, 13]
# w5 = init_weights([13, 13, 2, 2]) # out after (4, 4) max-pooling: [examples, filters, 3, 3]
# w6 = init_weights([13, 13 * 3 * 3])
#######################
conv_out1 = conv3d(
signals=X,
filters=w1
)
neurons = 625
kernel_size = 11
examples = data.shape[0]
if convolutional == 'y':
out_dim = data.shape[3] - (kernel_size - 1)
elif convolutional == 'n':
input_dim = data.shape[1]
batch_size = 1000
epochs = 100
n_batches = examples / batch_size
# create symbolic variables for the model
if convolutional == 'y':
input_ = t.ftensor4()
weights = init_weights((neurons, channels, kernel_size, kernel_size))
biases = theano.shared(np.asarray(np.zeros(neurons), dtype=theano.config.floatX))
inhibition = theano.shared(np.asarray(np.zeros(neurons), dtype=theano.config.floatX))
target = theano.shared(np.asarray(np.zeros((batch_size, neurons, out_dim, out_dim)), dtype=theano.config.floatX))
elif convolutional == 'n':
input_ = t.fmatrix()
weights = init_weights((neurons, input_dim))
biases = theano.shared(np.asarray(np.zeros(neurons), dtype=theano.config.floatX))
inhibition = theano.shared(np.asarray(np.zeros(neurons), dtype=theano.config.floatX))
target = theano.shared(np.asarray(np.zeros((batch_size, neurons)), dtype=theano.config.floatX))
print "building the model..."
model = ELPS(weights, biases, input_, inhibition, target, neurons, examples, batch_size, convolutional)
print "getting training functions..."
train = model.training_function()
train, _, _ = model_.training_functions(data)
# train the sparse filtering network
print "training network..."
for epoch in xrange(200): # 100
cost, weights = train[0](index=0)
print("Layer %i cost at epoch %i and batch %i: %f" %
(1, epoch, 0, cost))
elif convolutional == 'y':
print "setting up network for" + model
X = t.ftensor4()
Y = t.fmatrix()
w_out = init.init_weights((train_out[model].shape[1], 10))
py_x = final_layer(X, w_out)
y_x = t.argmax(py_x, axis=1)
cost = t.mean(t.nnet.categorical_crossentropy(py_x, Y))
params = [w_out]
updates = RMSprop(cost, params, lr=0.001)
print "compiling theano functions..."
train = theano.function(inputs=[X, Y],
outputs=[cost, w_out],
updates=updates,
allow_input_downcast=True)
predict = theano.function(inputs=[X],
outputs=y_x,
# define important variables
neurons = weights.shape[0]
dim = np.sqrt(weights.shape[1])
max_iter = 10000
# create data structures to save outputs
images_saved = np.zeros(weights.shape)
# loop over neurons
for neuron in xrange(neurons):
# # create symbolic variable for synthesized image
# image = t.fvector('image')
# create randomly initialized images with the same size as the neuron receptive fields
image = init.init_weights((1, weights.shape[1]))
# get the activation value of the neuron
activation = t.dot(model.layers[0].get_weights()[neuron], image.T) # using raw activation value
# _, _, _, _, activations, _ = outputs[model.n_layers - 1](image.get_value())
# activation = activations[neuron]
# todo: this will not scale easily to deep architecture; should pass to Network
# create cost function for evaluation
Lambda = .1 # parameter of l2 shrinkage
cost = -(activation[0] - Lambda * l2_norm(image)) # negative in front for gradient ascent
# put parameters into list
parameters = image
# get updates
