def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)):
"""
Allocate a LeNetConvPoolLayer with shared variable internal parameters.
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type input: theano.tensor.dtensor4
:param input: symbolic image tensor, of shape image_shape
:type filter_shape: tuple or list of length 4
:param filter_shape: (number of filters, num input feature maps,
filter height,filter width)
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
:type poolsize: tuple or list of length 2
:param poolsize: the downsampling (pooling) factor (#rows,#cols)
"""
assert image_shape[1] == filter_shape[1]
self.input = input
# there are "num input feature maps * filter height * filter width"
# inputs to each hidden unit
fan_in = numpy.prod(filter_shape[1:])
# each unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) /
numpy.prod(poolsize))
# initialize weights with random weights
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
self.W = theano.shared(numpy.asarray(rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX),
borrow=True)
# the bias is a 1D tensor -- one bias per output feature map
b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, borrow=True)
# convolve input feature maps with filters
# conv_out = conv.conv2d(input=input, filters=self.W,
# filter_shape=filter_shape, image_shape=image_shape)
input_shuffled = input.dimshuffle(1, 2, 3, 0) # bc01 to c01b
filters_shuffled = self.W.dimshuffle(1, 2, 3, 0) # bc01 to c01b
conv_op = FilterActs(stride=1, partial_sum=1)
contiguous_input = gpu_contiguous(input_shuffled)
contiguous_filters = gpu_contiguous(filters_shuffled)
conv_out_shuffled = conv_op(contiguous_input, contiguous_filters)
# downsample each feature map individually, using maxpooling
# pooled_out = downsample.max_pool_2d(input=conv_out,
# ds=poolsize, ignore_border=True)
pool_op = MaxPool(ds=poolsize[0], stride=poolsize[0])
pooled_out_shuffled = pool_op(conv_out_shuffled)
pooled_out = pooled_out_shuffled.dimshuffle(3, 0, 1, 2) # c01b to bc01
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1,n_filters,1,1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
# store parameters of this layer
self.params = [self.W, self.b]
def compilePredictActivation(self, net, layerNum):
variable = net.x if layerNum == 0 else net.varArrayAc[layerNum - 1]
#Calc shapes for reshape function on-the-fly. Assume we have square images as input.
sX = T.cast(T.sqrt(T.shape(variable)[0] / self.kernel_shape[1]),
'int32')
#Converts input from 2 to 4 dimensions
Xr = T.reshape(variable.T,
(T.shape(variable)[1], self.kernel_shape[1], sX, sX))
if self.optimized:
out_size = T.cast(
T.ceil((T.shape(Xr)[-1] -
T.shape(net.varWeights[layerNum]['w'])[-1] + 1) /
np.float32(self.stride)), 'int32')
conv_op = FilterActs(stride=self.stride)
input_shuffled = Xr.dimshuffle(1, 2, 3, 0) # bc01 to c01b
filters_shuffled = net.varWeights[layerNum]['w'].dimshuffle(
1, 2, 3, 0) # bc01 to c01b
filters_flipped = filters_shuffled[:, ::-1, ::
-1, :] # flip rows and columns
contiguous_input = gpu_contiguous(input_shuffled)
contiguous_filters = gpu_contiguous(
filters_flipped * (self.dropout if self.dropout else 1.0))
a = conv_op(contiguous_input, contiguous_filters)
a = a[:, :out_size, :out_size, :]
#Add bias
a = a + net.varWeights[layerNum]['b'].dimshuffle(0, 'x', 'x', 'x')
else:
a = T.nnet.conv2d(Xr,
net.varWeights[layerNum]['w'] *
(net.dropOutVectors[layerNum].dimshuffle(
'x', 'x', 0, 1) if self.dropout else 1.0),
border_mode='valid',
subsample=(self.stride, self.stride))
#Add bias
a = a + net.varWeights[layerNum]['b'].dimshuffle('x', 0, 'x', 'x')
if self.pooling:
if self.optimized:
#Pooling
# ds - side of square pool window
# stride - Defines the stride size between successive pooling squares.
# Setting this parameter smaller than sizeX produces overlapping pools.
# Setting it equal to sizeX gives the usual, non-overlapping pools. Values greater than sizeX are not allowed.
pool_op = MaxPool(ds=self.pooling_shape,
stride=self.pooling_shape)
contiguous_input = gpu_contiguous(
a.astype(theano.config.floatX))
a = pool_op(contiguous_input)
a = a.dimshuffle(3, 0, 1, 2) # c01b to bc01
else:
a = downsample.max_pool_2d(
a, (self.pooling_shape, self.pooling_shape),
ignore_border=False)
else:
if self.optimized:
a = a.dimshuffle(3, 0, 1, 2) # c01b to bc01
a = T.flatten(a, outdim=2).T
#Sigmoid
a = self.activation(a, self.pool_size)
net.varArrayAc.append(a)
def __init__(self,
rng,
input,
filter_shape,
image_shape,
poolsize=(2, 2),
cuda_convnet=0,
W=None,
b=None,
activation=T.tanh,
border_mode='valid',
partial_sum=1,
pad=0):
"""
Allocate a LeNetConvPoolLayer with shared variable internal parameters.
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type input: theano.tensor.dtensor4
:param input: symbolic image tensor, of shape image_shape
:type filter_shape: tuple or list of length 4
:param filter_shape: (number of filters, num input feature maps,
filter height, filter width)
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
:type poolsize: tuple or list of length 2
:param poolsize: the downsampling (pooling) factor (#rows, #cols)
"""
channels_idx = 0 if cuda_convnet else 1
assert image_shape[channels_idx] == filter_shape[channels_idx]
self.input = input
if W is None:
if cuda_convnet:
fan_in = numpy.prod(filter_shape[0:3])
fan_out = (filter_shape[3] * numpy.prod(filter_shape[1:3]) /
numpy.prod(poolsize))
# TODO: correct numpy.prod(poolsize). Theano's max_pool_2d uses a tuple
# to signify (x,y) poolsize and doesn't overlap these. cuda-convnet uses
# the tuple to signify a square pool of size x^2, with a stride y. So only
# if x == y in this case does numpy.prod(poolsize) work.
else:
# there are "num input feature maps * filter height * filter width"
# inputs to __each__ hidden unit
fan_in = numpy.prod(filter_shape[1:])
# __each__ unit in the lower layer receives a gradient from:
# "num output feature maps * filter height * filter width" /
# pooling size
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) /
numpy.prod(poolsize))
# initialize weights with random weights
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
W_values = numpy.asarray(rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX)
else:
W_values = numpy.asarray(W, dtype=theano.config.floatX)
# the bias is a 1D tensor -- one bias per output feature map
if b is None:
numchan = filter_shape[3] if cuda_convnet else filter_shape[0]
b_values = numpy.zeros((numchan, ), dtype=theano.config.floatX)
else:
b_values = numpy.asarray(b, dtype=theano.config.floatX)
self.W = theano.shared(value=W_values, borrow=True)
self.b = theano.shared(value=b_values, borrow=True)
# convolve input feature maps with filters
if cuda_convnet:
conv_op = FilterActs(partial_sum=partial_sum, pad=pad)
contiguous_input = gpu_contiguous(input)
contiguous_filters = gpu_contiguous(self.W)
conv_out = conv_op(contiguous_input, contiguous_filters)
else:
conv_out = conv.conv2d(input=input,
filters=self.W,
image_shape=image_shape,
filter_shape=filter_shape,
border_mode=border_mode)
# downsample each feature map individually, using maxpooling
if (poolsize[0] == 1
and cuda_convnet) or (poolsize[0] == 1 and poolsize[1] == 1
and not cuda_convnet):
pooled_out = conv_out
elif cuda_convnet:
pool_op = MaxPool(ds=poolsize[0], stride=poolsize[1])
contiguous_input = gpu_contiguous(conv_out)
pooled_out = pool_op(contiguous_input)
else:
pooled_out = downsample.max_pool_2d(input=conv_out,
ds=poolsize,
ignore_border=True)
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
if cuda_convnet:
self.output = activation(pooled_out +
self.b.dimshuffle(0, 'x', 'x', 'x'))
else:
self.output = activation(pooled_out +
self.b.dimshuffle('x', 0, 'x', 'x'))
# store parameters of this layer
self.params = [self.W, self.b]
def __init__(self,
numpy_rng=None,
input=None,
is_input_layer=False,
input_shape=(1, 28, 28),
filter_shape=(2, 1, 5, 5),
poolsize=(1, 1),
activation=T.tanh,
flatten=False,
border_mode='valid',
non_maximum_erasing=False,
W=None,
b=None):
assert input_shape[1] == filter_shape[1]
if is_input_layer:
self.input = input.reshape(input_shape).dimshuffle(1, 2, 3, 0)
else:
self.input = input
# Now reconstruct the input_shape and filter_shape
input_shape = (input_shape[1], input_shape[2], input_shape[3],
input_shape[0])
filter_shape = (filter_shape[1], filter_shape[2], filter_shape[3],
filter_shape[0])
self.input_shape = input_shape
self.filter_shape = filter_shape
self.poolsize = poolsize
self.activation = activation
self.flatten = flatten
fan_in = numpy.prod(filter_shape[:3])
fan_out = (filter_shape[3] * numpy.prod(filter_shape[1:3]) /
numpy.prod(poolsize))
# initialize weights with random weights
if W is None:
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
initial_W = numpy.asarray(numpy_rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX)
if activation == T.nnet.sigmoid:
initial_W *= 4
W = theano.shared(value=initial_W, name='W')
self.W = W
# the bias is a 1D tensor -- one bias per output feature map
if b is None:
b_values = numpy.zeros((filter_shape[3], ),
dtype=theano.config.floatX)
b = theano.shared(value=b_values, name='b')
self.b = b
# for momentum
self.delta_W = theano.shared(value=numpy.zeros(
filter_shape, dtype=theano.config.floatX),
name='delta_W')
self.delta_b = theano.shared(value=numpy.zeros_like(
self.b.get_value(borrow=True), dtype=theano.config.floatX),
name='delta_b')
# convolve input feature maps with filters
conv_op = FilterActs()
contiguous_input = gpu_contiguous(self.input)
contiguous_filters = gpu_contiguous(self.W)
conv_out = conv_op(contiguous_input, contiguous_filters)
y_out = activation(conv_out + self.b.dimshuffle(0, 'x', 'x', 'x'))
pool_op = MaxPool(ds=poolsize[0], stride=poolsize[0])
pooled_out = pool_op(y_out)
if non_maximum_erasing:
ds = tuple(poolsize)
po = pooled_out.repeat(ds[0], axis=2).repeat(ds[1], axis=3)
self.output = T.eq(y_out, po) * y_out
else:
self.output = pooled_out
if flatten:
self.output = self.output.dimshuffle(3, 0, 1, 2) # c01b to bc01
self.output = self.output.flatten(2)
self.params = [self.W, self.b]
self.delta_params = [self.delta_W, self.delta_b]
def __init__(
self,
input,
image_shape,
filter_shape,
convstride,
padsize,
group,
poolsize,
poolstride,
bias_init,
lrn=False,
lib_conv='cudnn',
):
'''
lib_conv can be cudnn (recommended)or cudaconvnet
'''
self.filter_size = filter_shape
self.convstride = convstride
self.padsize = padsize
self.poolsize = poolsize
self.poolstride = poolstride
self.channel = image_shape[0]
self.lrn = lrn
self.lib_conv = lib_conv
assert group in [1, 2]
self.filter_shape = np.asarray(filter_shape)
self.image_shape = np.asarray(image_shape)
if self.lrn:
self.lrn_func = CrossChannelNormalization()
if group == 1:
self.W = Weight(self.filter_shape)
self.b = Weight(self.filter_shape[3], bias_init, std=0)
else:
self.filter_shape[0] = self.filter_shape[0] / 2
self.filter_shape[3] = self.filter_shape[3] / 2
self.image_shape[0] = self.image_shape[0] / 2
self.image_shape[3] = self.image_shape[3] / 2
self.W0 = Weight(self.filter_shape)
self.W1 = Weight(self.filter_shape)
self.b0 = Weight(self.filter_shape[3], bias_init, std=0)
self.b1 = Weight(self.filter_shape[3], bias_init, std=0)
if lib_conv == 'cudaconvnet':
self.conv_op = FilterActs(pad=self.padsize,
stride=self.convstride,
partial_sum=1)
# Conv
if group == 1:
contiguous_input = gpu_contiguous(input)
contiguous_filters = gpu_contiguous(self.W.val)
conv_out = self.conv_op(contiguous_input, contiguous_filters)
conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x')
else:
contiguous_input0 = gpu_contiguous(input[:self.channel /
2, :, :, :])
contiguous_filters0 = gpu_contiguous(self.W0.val)
conv_out0 = self.conv_op(contiguous_input0,
contiguous_filters0)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle(0, 'x', 'x', 'x')
contiguous_input1 = gpu_contiguous(input[self.channel /
2:, :, :, :])
contiguous_filters1 = gpu_contiguous(self.W1.val)
conv_out1 = self.conv_op(contiguous_input1,
contiguous_filters1)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle(0, 'x', 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=0)
# ReLu
self.output = T.maximum(conv_out, 0)
conv_out = gpu_contiguous(conv_out)
# Pooling
if self.poolsize != 1:
self.pool_op = MaxPool(ds=poolsize, stride=poolstride)
self.output = self.pool_op(self.output)
elif lib_conv == 'cudnn':
input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01
# in01out to outin01
# print image_shape_shuffled
# print filter_shape_shuffled
if group == 1:
W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out = dnn.dnn_conv(
img=input_shuffled,
kerns=W_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
else:
W0_shuffled = \
self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out0 = \
dnn.dnn_conv(img=input_shuffled[:, :self.channel / 2,
:, :],
kerns=W0_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle('x', 0, 'x', 'x')
W1_shuffled = \
self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out1 = \
dnn.dnn_conv(img=input_shuffled[:, self.channel / 2:,
:, :],
kerns=W1_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle('x', 0, 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
# ReLu
self.output = T.maximum(conv_out, 0)
# Pooling
if self.poolsize != 1:
self.output = dnn.dnn_pool(self.output,
ws=(poolsize, poolsize),
stride=(poolstride, poolstride))
self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b
else:
NotImplementedError("lib_conv can only be cudaconvnet or cudnn")
# LRN
if self.lrn:
# lrn_input = gpu_contiguous(self.output)
self.output = self.lrn_func(self.output)
if group == 1:
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
else:
self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val]
self.weight_type = ['W', 'b', 'W', 'b']
print "conv ({}) layer with shape_in: {}".format(
lib_conv, str(image_shape))
def __init__(self, rng, input=None, filt_def=None, pool_def=(2, 2), \
activation=None, drop_rate=0., input_noise=0., bias_noise=0., \
W=None, b=None, name="", W_scale=1.0):
# Setup a shared random generator for this layer
#self.rng = theano.tensor.shared_randomstreams.RandomStreams( \
# rng.randint(100000))
self.rng = CURAND_RandomStreams(rng.randint(1000000))
self.clean_input = input
# Add gaussian noise to the input (if desired)
if (input_noise > 1e-4):
self.fuzzy_input = input + self.rng.normal(size=input.shape, \
avg=0.0, std=input_noise, dtype=theano.config.floatX)
else:
self.fuzzy_input = input
# Apply masking noise to the input (if desired)
if (drop_rate > 1e-4):
self.noisy_input = self._drop_from_input(self.fuzzy_input,
drop_rate)
else:
self.noisy_input = self.fuzzy_input
# Set the activation function for the conv filters
if activation:
self.activation = activation
else:
self.activation = lambda x: relu_actfun(x)
# initialize weights with random weights
W_init = 0.01 * np.asarray(rng.normal( \
size=filt_def), dtype=theano.config.floatX)
self.W = theano.shared(value=(W_scale*W_init), \
name="{0:s}_W".format(name))
# the bias is a 1D tensor -- one bias per output feature map
b_init = np.zeros((filt_def[0], ), dtype=theano.config.floatX) + 0.1
self.b = theano.shared(value=b_init, name="{0:s}_b".format(name))
# convolve input feature maps with filters
input_c01b = self.noisy_input.dimshuffle(1, 2, 3, 0) # bc01 to c01b
filters_c01b = self.W.dimshuffle(1, 2, 3, 0) # bc01 to c01b
conv_op = FilterActs(stride=1, partial_sum=1)
contig_input = gpu_contiguous(input_c01b)
contig_filters = gpu_contiguous(filters_c01b)
conv_out_c01b = conv_op(contig_input, contig_filters)
if (bias_noise > 1e-4):
noisy_conv_out_c01b = conv_out_c01b + self.rng.normal( \
size=conv_out_c01b.shape, avg=0.0, std=bias_noise, \
dtype=theano.config.floatX)
else:
noisy_conv_out_c01b = conv_out_c01b
# downsample each feature map individually, using maxpooling
pool_op = MaxPool(ds=pool_def[0], stride=pool_def[1])
mp_out_c01b = pool_op(noisy_conv_out_c01b)
mp_out_bc01 = mp_out_c01b.dimshuffle(3, 0, 1, 2) # c01b to bc01
# add the bias term. Since the bias is a vector (1D array), we first
# reshape it to a tensor of shape (1,n_filters,1,1). Each bias will
# thus be broadcasted across mini-batches and feature map
# width & height
self.noisy_linear_output = mp_out_bc01 + self.b.dimshuffle(
'x', 0, 'x', 'x')
self.linear_output = self.noisy_linear_output
self.output = self.activation(self.noisy_linear_output)
# store parameters of this layer
self.params = [self.W, self.b]
return
def test_pool():
try:
if hasattr(mode_with_gpu, 'check_isfinite'):
mode_with_gpu_check_is_finite_prev = mode_with_gpu.check_isfinite
if hasattr(mode_without_gpu, 'check_isfinite'):
mode_without_gpu_check_is_finite_prev = mode_without_gpu.check_isfinite
mode_with_gpu.check_isfinite = False
mode_without_gpu.check_isfinite = False
#(batch, channel, x, y)
shps = [
(1, 1, 2, 2),
(1, 1, 1, 1),
(1, 1, 4, 4),
(1, 2, 2, 2),
(1, 1, 4, 4),
(3, 1, 4, 4),
(1, 5, 4, 4),
(3, 5, 4, 4),
(25, 1, 7, 7),
(1, 1, 12, 12),
(1, 1, 14, 14),
(1, 1, 16, 16),
(1, 1, 18, 18),
(1, 1, 24, 24),
(1, 6, 24, 24),
(10, 1, 24, 24),
(10, 6, 24, 24),
(30, 6, 12, 12),
(30, 2, 24, 24),
(30, 6, 24, 24),
(65536, 1, 10, 10),
#(1, 65536, 10, 10),#crash as too much channel
(30, 3, 40, 40),
]
shps = [(channel, x, y, batch) for (batch, channel, x, y) in shps]
#numpy.random.RandomState(unittest_tools.fetch_seed()).shuffle(shps)
for shp in shps:
for ds in range(1, min(4, shp[2] + 1)):
# for start in range(shp[2] + 1):
for start in [0]:
for stride in range(1, min(shp[2], ds, 4) + 1):
print('test_pool shape=%s, ds=%d, stride=%d start=%d' %
(str(shp), ds, stride, start))
a = tcn.shared_constructor(my_rand(*shp), 'a')
op = MaxPool(ds=ds, stride=stride)
f = theano.function([], op(a), mode=mode_with_gpu)
assert any([
isinstance(node.op, MaxPool)
for node in f.maker.fgraph.toposort()
])
out = numpy.asarray(f())
#Compute the gold version with a Theano graph.
gold_out = gold_max_pool_c01b(a, (ds, ds),
(stride, stride),
shp[1:3])
f2 = theano.function([],
gold_out,
mode=mode_without_gpu)
assert not any([
isinstance(node.op, MaxPool)
for node in f2.maker.fgraph.toposort()
])
out2 = f2()
numpy.testing.assert_allclose(out,
out2,
err_msg=str(out - out2))
# grad testing
# The code support grad only in this case.
if shp[0] % 16 != 0:
shp2 = list(shp)
shp2[0] *= 16
# This make it crash due to not enough memory.
# On a GPU with 1279M of ram.
if numpy.prod(shp2) >= (16 * 10 * 10 * 65536):
continue
a.set_value(my_rand(*shp2))
g = theano.function([],
grad(op(a).sum(), a),
mode=mode_with_gpu)
g2 = theano.function([],
grad(gold_out.sum(), a),
mode=mode_without_gpu)
assert any([
isinstance(node.op, MaxPoolGrad)
for node in g.maker.fgraph.toposort()
])
assert not any([
isinstance(node.op, MaxPoolGrad)
for node in g2.maker.fgraph.toposort()
])
numpy.testing.assert_allclose(g(),
g2(),
err_msg=str(shp))
# Don't call verify_grad. There was problem with
# the test and we already assert that 2 version
# are equals. Also, it will be slower to verify
# like that then the comparison.
continue
theano.tests.unittest_tools.verify_grad(
op, [a.get_value()])
finally:
if 'mode_with_gpu_check_is_finite_prev' in locals():
mode_with_gpu.check_isfinite = mode_with_gpu_check_is_finite_prev
if 'mode_without_gpu_check_is_finite_prev' in locals():
mode_without_gpu.check_isfinite = mode_without_gpu_check_is_finite_prev
def __init__(self,
numpy_rng=None,
input=None,
input_shape=(256, 1, 28, 28),
filter_shape=(2, 1, 5, 5),
poolsize=(1, 1),
activation=T.tanh,
flatten=False,
border_mode='valid',
non_maximum_erasing=False,
W=None,
b=None,
use_fast=False,
testing=False):
self.type = 'conv'
assert input_shape[1] == filter_shape[1]
if testing:
self.input = input.reshape((input.shape[0], input_shape[1],
input_shape[2], input_shape[3]))
input_shape = None
else:
self.input = input.reshape(input_shape)
self.filter_shape = filter_shape
self.poolsize = poolsize
self.activation = activation
self.flatten = flatten
fan_in = numpy.prod(filter_shape[1:])
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) /
numpy.prod(poolsize))
# initialize weights with random weights
if W is None:
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
initial_W = numpy.asarray(numpy_rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX)
if activation == T.nnet.sigmoid:
initial_W *= 4
W = theano.shared(value=initial_W, name='W')
self.W = W
# the bias is a 1D tensor -- one bias per output feature map
if b is None:
b_values = numpy.zeros((filter_shape[0], ),
dtype=theano.config.floatX)
b = theano.shared(value=b_values, name='b')
self.b = b
# for momentum
self.delta_W = theano.shared(value=numpy.zeros(
filter_shape, dtype=theano.config.floatX),
name='delta_W')
self.delta_b = theano.shared(value=numpy.zeros_like(
self.b.get_value(borrow=True), dtype=theano.config.floatX),
name='delta_b')
# convolve input feature maps with filters
if use_fast:
from theano.sandbox.cuda.basic_ops import gpu_contiguous
from pylearn2.sandbox.cuda_convnet.filter_acts import FilterActs
from pylearn2.sandbox.cuda_convnet.pool import MaxPool
input_shuffled = self.input.dimshuffle(1, 2, 3, 0) # bc01 to c01b
filters_shuffled = self.W.dimshuffle(1, 2, 3, 0) # bc01 to c01b
conv_op = FilterActs()
contiguous_input = gpu_contiguous(input_shuffled)
contiguous_filters = gpu_contiguous(filters_shuffled)
conv_out_shuffled = conv_op(contiguous_input, contiguous_filters)
y_out_shuffled = activation(conv_out_shuffled +
self.b.dimshuffle(0, 'x', 'x', 'x'))
pool_op = MaxPool(ds=poolsize[0], stride=poolsize[0])
pooled_out = pool_op(y_out_shuffled).dimshuffle(3, 0, 1, 2)
else:
conv_out = conv.conv2d(input=self.input,
filters=self.W,
filter_shape=filter_shape,
image_shape=input_shape,
border_mode=border_mode)
y_out = activation(conv_out + self.b.dimshuffle('x', 0, 'x', 'x'))
# downsample each feature map individually, using maxpooling
pooled_out = downsample.max_pool_2d(input=y_out,
ds=poolsize,
ignore_border=True)
if non_maximum_erasing:
ds = tuple(poolsize)
po = pooled_out.repeat(ds[0], axis=2).repeat(ds[1], axis=3)
self.output = T.eq(y_out, po) * y_out
else:
self.output = pooled_out
if flatten:
self.output = self.output.flatten(2)
self.params = [self.W, self.b]
self.delta_params = [self.delta_W, self.delta_b]
def __init__(self, input, image_shape, filter_shape, convstride, padsize,
group, poolsize, poolstride, bias_init, lrn=False,
lib_conv='cudnn',
):
"""
lib_conv can be cudnn or cudaconvet
"""
assert group in [1, 2]
assert lib_conv in ['cudnn', 'cudaconvnet']
self.filter_size = filter_shape
self.convstride = convstride
self.padsize = padsize
self.poolsize = poolsize
self.poolstride = poolstride
if lib_conv == 'cudnn':
self.channel = image_shape[1]
else:
self.channel = image_shape[0]
self.lrn = lrn
self.lib_conv = lib_conv
self.filter_shape = np.asarray(filter_shape)
self.image_shape = np.asarray(image_shape)
if self.lrn:
self.lrn_func = CrossChannelNormalization()
if group == 1:
self.W = Weight(self.filter_shape)
if lib_conv == 'cudnn':
self.b = Weight(self.filter_shape[0], bias_init, std=0)
else:
self.b = Weight(self.filter_shape[3], bias_init, std=0)
else:
if lib_conv == 'cudnn':
self.filter_shape[1] /= 2
self.filter_shape[0] /= 2
self.image_shape[1] /= 2
self.image_shape[0] /= 2
else:
self.filter_shape[0] /= 2
self.filter_shape[3] /= 2
self.image_shape[0] /= 2
self.image_shape[3] /= 2
self.W0 = Weight(self.filter_shape)
self.W1 = Weight(self.filter_shape)
if lib_conv=='cudnn':
self.b0 = Weight(self.filter_shape[0], bias_init, std=0)
self.b1 = Weight(self.filter_shape[0], bias_init, std=0)
else:
self.b0 = Weight(self.filter_shape[3], bias_init, std=0)
self.b1 = Weight(self.filter_shape[3], bias_init, std=0)
if lib_conv == 'cudaconvnet':
self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride,
partial_sum=1)
# Conv
if group == 1:
contiguous_input = gpu_contiguous(input)
contiguous_filters = gpu_contiguous(self.W.val)
conv_out = self.conv_op(contiguous_input, contiguous_filters)
conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x')
else:
contiguous_input0 = gpu_contiguous(
input[:self.channel / 2, :, :, :])
contiguous_filters0 = gpu_contiguous(self.W0.val)
conv_out0 = self.conv_op(
contiguous_input0, contiguous_filters0)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle(0, 'x', 'x', 'x')
contiguous_input1 = gpu_contiguous(
input[self.channel / 2:, :, :, :])
contiguous_filters1 = gpu_contiguous(self.W1.val)
conv_out1 = self.conv_op(
contiguous_input1, contiguous_filters1)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle(0, 'x', 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=0)
# ReLu
conv_out = T.maximum(conv_out, 0)
self.output = gpu_contiguous(conv_out)
# Pooling
if self.poolsize != 1:
self.pool_op = MaxPool(ds=poolsize, stride=poolstride)
self.output = self.pool_op(self.output)
else:
if group == 1:
conv_out = dnn.dnn_conv(img=input,
kerns=self.W.val,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
else:
input1, input2 = \
theano.tensor.split(input,
[self.channel/2, self.channel/2],
2, axis=1)
conv_out0 = \
dnn.dnn_conv(img=input1,
kerns=self.W0.val,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out0 = conv_out0 + self.b0.val.dimshuffle('x', 0, 'x', 'x')
conv_out1 = \
dnn.dnn_conv(img=input2,
kerns=self.W1.val,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out1 = conv_out1 + self.b1.val.dimshuffle('x', 0, 'x', 'x')
# self.conv_out = conv_out1
conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
# ReLu
self.output = T.maximum(conv_out, 0)
# Pooling
if self.poolsize != 1:
self.output = dnn.dnn_pool(self.output,
ws=(poolsize, poolsize),
stride=(poolstride, poolstride))
# LRN
if self.lrn:
self.output = self.lrn_func(self.output)
if group == 1:
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
else:
self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val]
self.weight_type = ['W', 'b', 'W', 'b']
print "conv ({}) layer with shape_in: {}".format(lib_conv,
str(image_shape))
def __init__(self,
numpy_rng=None,
input=None,
filter_shape=(2, 1, 5, 5),
poolsize=(1, 1),
activation=T.nnet.sigmoid,
flatten=False,
use_fast=False):
self.type = 'conv'
self.input = input
self.filter_shape = filter_shape
self.poolsize = poolsize
self.activation = activation
self.flatten = flatten
fan_in = numpy.prod(filter_shape[1:])
fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) /
numpy.prod(poolsize))
# initialize weights with random weights
W_bound = numpy.sqrt(6. / (fan_in + fan_out))
initial_W = numpy.asarray(numpy_rng.uniform(low=-W_bound,
high=W_bound,
size=filter_shape),
dtype=theano.config.floatX)
if activation == T.nnet.sigmoid:
initial_W *= 4
W = theano.shared(value=initial_W, name='W')
self.W = W
# the bias is a 1D tensor -- one bias per output feature map
b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX)
self.b = theano.shared(value=b_values, name='b')
# convolve input feature maps with filters
if use_fast:
from theano.sandbox.cuda.basic_ops import gpu_contiguous
from pylearn2.sandbox.cuda_convnet.filter_acts import FilterActs
from pylearn2.sandbox.cuda_convnet.pool import MaxPool
input_shuffled = self.input.dimshuffle(1, 2, 3, 0) # bc01 to c01b
filters_shuffled = self.W.dimshuffle(1, 2, 3, 0) # bc01 to c01b
conv_op = FilterActs()
contiguous_input = gpu_contiguous(input_shuffled)
contiguous_filters = gpu_contiguous(filters_shuffled)
conv_out_shuffled = conv_op(contiguous_input, contiguous_filters)
y_out_shuffled = activation(conv_out_shuffled +
self.b.dimshuffle(0, 'x', 'x', 'x'))
pool_op = MaxPool(ds=poolsize[0], stride=poolsize[0])
self.output = pool_op(y_out_shuffled).dimshuffle(3, 0, 1, 2)
else:
conv_out = conv.conv2d(input=self.input,
filters=self.W,
filter_shape=filter_shape)
y_out = activation(conv_out + self.b.dimshuffle('x', 0, 'x', 'x'))
# downsample each feature map individually, using maxpooling
self.output = downsample.max_pool_2d(input=y_out,
ds=poolsize,
ignore_border=True)
if self.flatten:
self.output = self.output.flatten(2)
self.params = [self.W, self.b]
def __init__(self,
input,
image_shape,
filter_shape,
convstride,
padsize,
group,
poolsize,
poolstride,
bias_init,
lrn=False):
self.filter_size = filter_shape
self.convstride = convstride
self.padsize = padsize
self.poolsize = poolsize
self.poolstride = poolstride
self.channel = image_shape[0]
self.lrn = lrn
assert group in [1, 2]
self.filter_shape = np.asarray(filter_shape)
self.image_shape = np.asarray(image_shape)
if self.lrn:
self.lrn_func = CrossChannelNormalization()
if group == 1:
self.W = Weight(self.filter_shape)
self.b = Weight(self.filter_shape[3], bias_init, std=0)
else:
self.filter_shape[0] = self.filter_shape[0] / 2
self.filter_shape[3] = self.filter_shape[3] / 2
self.image_shape[0] = self.image_shape[0] / 2
self.image_shape[3] = self.image_shape[3] / 2
self.W0 = Weight(self.filter_shape)
self.W1 = Weight(self.filter_shape)
self.b0 = Weight(self.filter_shape[3], bias_init, std=0)
self.b1 = Weight(self.filter_shape[3], bias_init, std=0)
self.conv_op = FilterActs(pad=self.padsize,
stride=self.convstride,
partial_sum=1)
# Conv
if group == 1:
contiguous_input = gpu_contiguous(input)
contiguous_filters = gpu_contiguous(self.W.val)
conv_out = self.conv_op(contiguous_input, contiguous_filters)
conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x')
else:
contiguous_input0 = gpu_contiguous(input[:self.channel /
2, :, :, :])
contiguous_filters0 = gpu_contiguous(self.W0.val)
conv_out0 = self.conv_op(contiguous_input0, contiguous_filters0)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle(0, 'x', 'x', 'x')
contiguous_input1 = gpu_contiguous(input[self.channel /
2:, :, :, :])
contiguous_filters1 = gpu_contiguous(self.W1.val)
conv_out1 = self.conv_op(contiguous_input1, contiguous_filters1)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle(0, 'x', 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=0)
# ReLu
self.output = T.maximum(conv_out, 0)
conv_out = gpu_contiguous(conv_out)
# Pooling
if self.poolsize != 1:
self.pool_op = MaxPool(ds=poolsize, stride=poolstride)
self.output = self.pool_op(self.output)
# LRN
if self.lrn:
# lrn_input = gpu_contiguous(self.output)
self.output = self.lrn_func(self.output)
if group == 1:
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
else:
self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val]
self.weight_type = ['W', 'b', 'W', 'b']
print "conv layer with shape_in: " + str(image_shape)
def __init__(self, input, image_shape, filter_shape, convstride, padsize,
group, poolsize, poolstride, bias_init, lrn=False,
lib_conv='cudnn',
verbose=False
):
'''
lib_conv can be cudnn (recommended)or cudaconvnet
'''
self.filter_size = filter_shape
self.convstride = convstride
self.padsize = padsize
self.poolsize = poolsize
self.poolstride = poolstride
self.channel = image_shape[0]
self.lrn = lrn
self.lib_conv = lib_conv
self.verbose = verbose
assert group in [1, 2]
self.filter_shape = np.asarray(filter_shape)
self.image_shape = np.asarray(image_shape)
if self.lrn:
self.lrn_func = CrossChannelNormalization()
if group == 1:
self.W = Weight(self.filter_shape)
self.b = Weight(self.filter_shape[3], bias_init, std=0)
else:
self.filter_shape[0] = self.filter_shape[0] / 2
self.filter_shape[3] = self.filter_shape[3] / 2
self.image_shape[0] = self.image_shape[0] / 2
self.image_shape[3] = self.image_shape[3] / 2
self.W0 = Weight(self.filter_shape)
self.W1 = Weight(self.filter_shape)
self.b0 = Weight(self.filter_shape[3], bias_init, std=0)
self.b1 = Weight(self.filter_shape[3], bias_init, std=0)
if lib_conv == 'cudaconvnet':
self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride,
partial_sum=1)
from theano.sandbox.cuda.basic_ops import gpu_contiguous
# Conv
if group == 1:
contiguous_input = gpu_contiguous(input)
contiguous_filters = gpu_contiguous(self.W.val)
conv_out = self.conv_op(contiguous_input, contiguous_filters)
conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x')
else:
contiguous_input0 = gpu_contiguous(
input[:self.channel / 2, :, :, :])
contiguous_filters0 = gpu_contiguous(self.W0.val)
conv_out0 = self.conv_op(
contiguous_input0, contiguous_filters0)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle(0, 'x', 'x', 'x')
contiguous_input1 = gpu_contiguous(
input[self.channel / 2:, :, :, :])
contiguous_filters1 = gpu_contiguous(self.W1.val)
conv_out1 = self.conv_op(
contiguous_input1, contiguous_filters1)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle(0, 'x', 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=0)
# ReLu
conv_out = gpu_contiguous(conv_out)
self.output = T.maximum(conv_out, 0)
# Pooling
if self.poolsize != 1:
self.pool_op = MaxPool(ds=poolsize, stride=poolstride)
self.output = self.pool_op(self.output)
elif lib_conv == 'corrmm':
from theano.sandbox.cuda.basic_ops import gpu_contiguous
from theano.sandbox.cuda.blas import GpuCorrMM
border_mode = 'half' if padsize == (filter_shape[1]-1)/2 else (padsize, padsize)
self.corr_mm_op = GpuCorrMM(subsample=(convstride,convstride),
border_mode=border_mode)
flip_filters=True
input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01
if group==1:
filters = self.W.val.dimshuffle(3, 0, 1, 2)
if flip_filters:
filters = filters[:, :, ::-1, ::-1] # flip top-down, left-right
contiguous_filters = gpu_contiguous(filters)
contiguous_input = gpu_contiguous(input_shuffled)
conv_out = self.corr_mm_op(contiguous_input, contiguous_filters)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
else:
W0_shuffled = \
self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
if flip_filters:
W0_shuffled = W0_shuffled[:, :, ::-1, ::-1]
contiguous_filters0 = gpu_contiguous(W0_shuffled)
contiguous_input0 = gpu_contiguous(input_shuffled[:, :self.channel / 2,:, :])
conv_out0 = self.corr_mm_op(contiguous_input0, contiguous_filters0)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle('x', 0, 'x', 'x')
W1_shuffled = \
self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
if flip_filters:
W1_shuffled = W1_shuffled[:, :, ::-1, ::-1]
contiguous_filters1 = gpu_contiguous(W1_shuffled)
contiguous_input1 = gpu_contiguous(input_shuffled[:, self.channel / 2:,:, :])
conv_out1 = self.corr_mm_op(contiguous_input1, contiguous_filters1)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle('x', 0, 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
# ReLu
self.output = T.maximum(conv_out, 0)
# Pooling
if self.poolsize != 1:
from theano.tensor.signal import downsample
self.output = downsample.max_pool_2d(self.output,
ds=(poolsize,poolsize),
st=(poolstride,poolstride),
ignore_border=False,
padding=(0,0),
mode='max',
)
self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b
elif lib_conv == 'cudnn':
input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01
# in01out to outin01
# print image_shape_shuffled
# print filter_shape_shuffled
if group == 1:
W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out = dnn.dnn_conv(img=input_shuffled,
kerns=W_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
else:
W0_shuffled = \
self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out0 = \
dnn.dnn_conv(img=input_shuffled[:, :self.channel / 2,
:, :],
kerns=W0_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle('x', 0, 'x', 'x')
W1_shuffled = \
self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out1 = \
dnn.dnn_conv(img=input_shuffled[:, self.channel / 2:,
:, :],
kerns=W1_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle('x', 0, 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
# ReLu
self.output = T.maximum(conv_out, 0)
# Pooling
if self.poolsize != 1:
self.output = dnn.dnn_pool(self.output,
ws=(poolsize, poolsize),
stride=(poolstride, poolstride))
self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b
else:
NotImplementedError("lib_conv can only be cudaconvnet or cudnn")
# LRN
if self.lrn:
# lrn_input = gpu_contiguous(self.output)
self.output = self.lrn_func(self.output)
if group == 1:
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
else:
self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val]
self.weight_type = ['W', 'b', 'W', 'b']
if self.verbose:
print "conv ({}) layer with shape_in: {}".format(lib_conv,
str(image_shape))
def conv(data,
filter_gen,
feature_batch_size,
num_feature_batches,
data_batch_size,
cuda_convnet=True,
symmetric_relu=True,
start_feature_batch=0,
pool_type='avg',
pool_size=14,
pad=0,
bias=1.0,
ps=6):
cuda_convnet = True
outX = int(math.ceil((data.shape[2] - ps + 1) / float(pool_size)))
outY = int(math.ceil((data.shape[3] - ps + 1) / float(pool_size)))
outFilters = feature_batch_size * num_feature_batches
if (symmetric_relu):
outFilters = 2 * outFilters
print "Out Shape ", outX, "x", outY, "x", outFilters
XFinal = np.zeros((data.shape[0], outFilters, outX, outY), 'float32')
filters = []
numImages = data.shape[0]
# Convert to cuda-convnet order
if (cuda_convnet):
data = data.transpose(1, 2, 3, 0)
# POOL OP CREATION
if (cuda_convnet):
if (pool_type == 'avg'):
pool_op = AvgPool(ds=pool_size, stride=pool_size)
elif (pool_type == 'max'):
pool_op = MaxPool(ds=pool_size, stride=pool_size)
else:
raise Exception('Unsupported pool type')
else:
pool_op = lambda X: T.signal.pool.pool_2d(
X, (pool_size, pool_size), ignore_border=False, mode='max')
if (cuda_convnet):
conv_op = FilterActs(pad=pad)
else:
conv_op = lambda X, F: T.nnet.conv2d(X, F)
CHANNEL_AXIS = 1
for j in range(num_feature_batches):
F = filter_gen(feature_batch_size)
if (cuda_convnet):
F = F.transpose(1, 2, 3, 0)
CHANNEL_AXIS = 0
filters.append(F)
FTheano = shared(F.astype('float32'))
start_filters = j * feature_batch_size
end_filters = (j + 1) * feature_batch_size
if symmetric_relu:
start_filters *= 2
end_filters *= 2
for i in range(int(np.ceil(numImages / float(data_batch_size)))):
start = i * data_batch_size
end = min((i + 1) * data_batch_size, numImages)
print "FEATURE BATCH #", (
j + start_feature_batch
), "DATA BATCH #", i, " SIZE IS ", end - start
if (cuda_convnet):
XBlock = shared(data[:, :, :, start:end])
else:
XBlock = shared(data[start:end, :, :, :])
if (cuda_convnet):
XBlock_gpu = gpu_contiguous(XBlock)
FTheano_gpu = gpu_contiguous(FTheano)
# CONV
XBlock_conv_out = conv_op(XBlock_gpu, FTheano_gpu)
# RELU
XBlock0 = T.nnet.relu(XBlock_conv_out - bias, 0)
if (symmetric_relu):
XBlock1 = T.nnet.relu(-1.0 * XBlock_conv_out - bias, 0)
XBlock0 = pool_op(XBlock0)
if (symmetric_relu):
XBlock1 = pool_op(XBlock1)
XBlockOut = np.concatenate((XBlock0.eval(), XBlock1.eval()),
axis=CHANNEL_AXIS)
else:
XBlockOut = np.array(XBlock0.eval())
if (cuda_convnet):
XBlockOut = XBlockOut.transpose(3, 0, 1, 2)
F = F.transpose(3, 0, 1, 2)
XBlock.set_value([[[[]]]])
XFinal[start:end, start_filters:end_filters, :, :] = XBlockOut
FTheano.set_value([[[[]]]])
filters = np.concatenate(filters, axis=0)
return (XFinal, filters)
#Calc avg activation for convolution results
sprs = 0
if sparsity:
sparse = T.mean(res, axis=(1, 2, 3))
epsilon = 1e-20
sparse = T.clip(sparse, epsilon, 1 - epsilon)
KL = T.sum(sparsityParam * T.log(sparsityParam / sparse) +
(1 - sparsityParam) * T.log((1 - sparsityParam) / (1 - sparse)))
sprs = KL * beta
#Pooling
#res = downsample.max_pool_2d(res, pool_shape, ignore_border=True)
#res_p = downsample.max_pool_2d(res_p, pool_shape, ignore_border=True)
pool_op = MaxPool(ds=2, stride=2)
contiguous_input = gpu_contiguous(res)
res = pool_op(contiguous_input)
contiguous_input_p = gpu_contiguous(res_p)
res_p = pool_op(contiguous_input_p)
res = res.dimshuffle(3, 0, 1, 2) # c01b to bc01
res_p = res_p.dimshuffle(3, 0, 1, 2) # c01b to bc01
#Separate function if U want to estimate output just after pooling
#cnn = theano.function(inputs=[X], outputs=res, allow_input_downcast=True)
#------#
CV_size = 6000