def test_convolutional_sequence():
x = tensor.tensor4('x')
num_channels = 4
pooling_size = 3
batch_size = 5
act = Rectifier()
conv = Convolutional((3, 3), 5, weights_init=Constant(1.),
biases_init=Constant(5.))
pooling = MaxPooling(pooling_size=(pooling_size, pooling_size))
conv2 = Convolutional((2, 2), 4, weights_init=Constant(1.))
seq = ConvolutionalSequence([conv, act, pooling.apply, conv2.apply, act],
num_channels, image_size=(17, 13))
seq.push_allocation_config()
assert conv.num_channels == 4
assert conv2.num_channels == 5
conv2.use_bias = False
y = seq.apply(x)
seq.initialize()
func = function([x], y)
x_val = numpy.ones((batch_size, 4, 17, 13), dtype=theano.config.floatX)
y_val = (numpy.ones((batch_size, 4, 4, 2)) *
(9 * 4 + 5) * 4 * 5)
assert_allclose(func(x_val), y_val)
def test_tied_biases():
x = tensor.tensor4('x')
num_channels = 4
num_filters = 3
batch_size = 5
filter_size = (3, 3)
conv = Convolutional(filter_size, num_filters, num_channels,
weights_init=Constant(1.), biases_init=Constant(2.),
tied_biases=True)
conv.initialize()
y = conv.apply(x)
func = function([x], y)
# Tied biases allows to pass images of different sizes
x_val_1 = numpy.ones((batch_size, num_channels, 10,
12), dtype=theano.config.floatX)
x_val_2 = numpy.ones((batch_size, num_channels, 23,
19), dtype=theano.config.floatX)
assert_allclose(func(x_val_1),
numpy.prod(filter_size) * num_channels *
numpy.ones((batch_size, num_filters, 8, 10)) + 2)
assert_allclose(func(x_val_2),
numpy.prod(filter_size) * num_channels *
numpy.ones((batch_size, num_filters, 21, 17)) + 2)
def create_KmaxPooling_cnn(layer0_input, embedding_size, input_len, config, pref):
'''
One layer convolution with different filter-sizes and maxpooling
'''
k = int(config[pref + '_kpool'])
filter_width_list = [int(fw) for fw in config[pref + '_filterwidth'].split()]
print filter_width_list
num_filters = int(config[pref+'_num_filters'])
#num_filters /= len(filter_width_list)
totfilters = 0
print input_len, embedding_size, num_filters
for i, fw in enumerate(filter_width_list):
num_feature_map = input_len - fw + 1 #39
conv = Convolutional(
image_size=(input_len, embedding_size),
filter_size=(fw, embedding_size),
num_filters=min(int(config[pref + '_maxfilter']), num_filters * fw),
num_channels=1
)
totfilters += conv.num_filters * k
# initialize2(conv, num_feature_map)
initialize([conv])
conv.name = pref + 'conv_' + str(fw)
layer0_input = debug_print(layer0_input, 'inp', False)
convout = conv.apply(layer0_input)
convout = debug_print(convout, 'convout', False)
kpoolout = KmaxPooling(convout, k).apply().flatten(2)
kpoolout = debug_print(kpoolout, 'poolout', False)
if i == 0:
outpools = kpoolout
else:
outpools = T.concatenate([outpools, kpoolout], axis=1)
name_rep_len = totfilters
return outpools, name_rep_len
def __init__(self,
filter_size,
num_filters,
num_channels,
noise_batch_size,
image_size=(None, None),
step=(1, 1),
border_mode='valid',
tied_biases=True,
prior_mean=0,
prior_noise_level=0,
**kwargs):
self.convolution = Convolutional()
self.mask = Convolutional(name='mask')
children = [self.convolution, self.mask]
kwargs.setdefault('children', []).extend(children)
super(NoisyConvolutional, self).__init__(**kwargs)
self.filter_size = filter_size
self.num_filters = num_filters
self.num_channels = num_channels
self.noise_batch_size = noise_batch_size
self.image_size = image_size
self.step = step
self.border_mode = border_mode
self.tied_biases = tied_biases
self.prior_mean = prior_mean
self.prior_noise_level = prior_noise_level
def create_kim_cnn(layer0_input, embedding_size, input_len, config, pref):
'''
One layer convolution with different filter-sizes and maxpooling
'''
filter_width_list = [int(fw) for fw in config[pref + '_filterwidth'].split()]
print filter_width_list
num_filters = int(config[pref+'_num_filters'])
#num_filters /= len(filter_width_list)
totfilters = 0
for i, fw in enumerate(filter_width_list):
num_feature_map = input_len - fw + 1 #39
conv = Convolutional(
image_size=(input_len, embedding_size),
filter_size=(fw, embedding_size),
num_filters=min(int(config[pref + '_maxfilter']), num_filters * fw),
num_channels=1
)
totfilters += conv.num_filters
initialize2(conv, num_feature_map)
conv.name = pref + 'conv_' + str(fw)
convout = conv.apply(layer0_input)
pool_layer = MaxPooling(
pooling_size=(num_feature_map,1)
)
pool_layer.name = pref + 'pool_' + str(fw)
act = Rectifier()
act.name = pref + 'act_' + str(fw)
outpool = act.apply(pool_layer.apply(convout)).flatten(2)
if i == 0:
outpools = outpool
else:
outpools = T.concatenate([outpools, outpool], axis=1)
name_rep_len = totfilters
return outpools, name_rep_len
def build_conv_layers(self, image=None):
if image is None:
image = T.ftensor4('spectrogram')
else:
image = image
conv_list = []
for layer in range(self.layers):
layer_param = self.params[layer]
conv_layer = Convolutional(layer_param[0], layer_param[1],
layer_param[2])
pool_layer = MaxPooling(layer_param[3])
conv_layer.name = "convolution" + str(layer)
pool_layer.name = "maxpooling" + str(layer)
conv_list.append(conv_layer)
conv_list.append(pool_layer)
conv_list.append(Rectifier())
conv_seq = ConvolutionalSequence(conv_list,
self.params[0][2],
image_size=self.image_size,
weights_init=IsotropicGaussian(
std=0.5, mean=0),
biases_init=Constant(0))
conv_seq._push_allocation_config()
conv_seq.initialize()
out = conv_seq.apply(image)
return out, conv_seq.get_dim('output')
def test_tied_biases():
x = tensor.tensor4('x')
num_channels = 4
num_filters = 3
batch_size = 5
filter_size = (3, 3)
# Tied biases are the default
conv = Convolutional(filter_size,
num_filters,
num_channels,
weights_init=Constant(1.),
biases_init=Constant(2.))
conv.initialize()
y = conv.apply(x)
func = function([x], y)
# Tied biases only provide one bias for each filter
assert_allclose(conv.b.eval().shape, (3, ))
# Tied biases allows to pass images of different sizes
x_val_1 = numpy.ones((batch_size, num_channels, 10, 12),
dtype=theano.config.floatX)
x_val_2 = numpy.ones((batch_size, num_channels, 23, 19),
dtype=theano.config.floatX)
assert_allclose(
func(x_val_1),
numpy.prod(filter_size) * num_channels * numpy.ones(
(batch_size, num_filters, 8, 10)) + 2)
assert_allclose(
func(x_val_2),
numpy.prod(filter_size) * num_channels * numpy.ones(
(batch_size, num_filters, 21, 17)) + 2)
def build_conv_layers(self, image=None) :
if image is None :
image = T.ftensor4('spectrogram')
else :
image = image
conv_list = []
for layer in range(self.layers) :
layer_param = self.params[layer]
conv_layer = Convolutional(layer_param[0], layer_param[1], layer_param[2])
pool_layer = MaxPooling(layer_param[3])
conv_layer.name = "convolution"+str(layer)
pool_layer.name = "maxpooling"+str(layer)
conv_list.append(conv_layer)
conv_list.append(pool_layer)
conv_list.append(Rectifier())
conv_seq = ConvolutionalSequence(
conv_list,
self.params[0][2],
image_size=self.image_size,
weights_init=IsotropicGaussian(std=0.5, mean=0),
biases_init=Constant(0))
conv_seq._push_allocation_config()
conv_seq.initialize()
out = conv_seq.apply(image)
return out, conv_seq.get_dim('output')
def test_convolutional_sequence():
x = tensor.tensor4('x')
num_channels = 4
pooling_size = 3
batch_size = 5
act = Rectifier()
conv = Convolutional((3, 3),
5,
weights_init=Constant(1.),
biases_init=Constant(5.))
pooling = MaxPooling(pooling_size=(pooling_size, pooling_size))
conv2 = Convolutional((2, 2), 4, weights_init=Constant(1.))
seq = ConvolutionalSequence([conv, act, pooling.apply, conv2.apply, act],
num_channels,
image_size=(17, 13))
seq.push_allocation_config()
assert conv.num_channels == 4
assert conv2.num_channels == 5
conv2.use_bias = False
y = seq.apply(x)
seq.initialize()
func = function([x], y)
x_val = numpy.ones((batch_size, 4, 17, 13), dtype=theano.config.floatX)
y_val = (numpy.ones((batch_size, 4, 4, 2)) * (9 * 4 + 5) * 4 * 5)
assert_allclose(func(x_val), y_val)
def create_kim_cnn(layer0_input, embedding_size, input_len, config, pref):
'''
One layer convolution with different filter-sizes and maxpooling
'''
filter_width_list = [
int(fw) for fw in config[pref + '_filterwidth'].split()
]
print filter_width_list
num_filters = int(config[pref + '_num_filters'])
#num_filters /= len(filter_width_list)
totfilters = 0
for i, fw in enumerate(filter_width_list):
num_feature_map = input_len - fw + 1 #39
conv = Convolutional(image_size=(input_len, embedding_size),
filter_size=(fw, embedding_size),
num_filters=min(int(config[pref + '_maxfilter']),
num_filters * fw),
num_channels=1)
totfilters += conv.num_filters
initialize2(conv, num_feature_map)
conv.name = pref + 'conv_' + str(fw)
convout = conv.apply(layer0_input)
pool_layer = MaxPooling(pooling_size=(num_feature_map, 1))
pool_layer.name = pref + 'pool_' + str(fw)
act = Rectifier()
act.name = pref + 'act_' + str(fw)
outpool = act.apply(pool_layer.apply(convout)).flatten(2)
if i == 0:
outpools = outpool
else:
outpools = T.concatenate([outpools, outpool], axis=1)
name_rep_len = totfilters
return outpools, name_rep_len
def test_convolutional_sequence_tied_biases_pushed_if_explicitly_set():
cnn = ConvolutionalSequence(sum([[
Convolutional(filter_size=(1, 1), num_filters=1, tied_biases=True),
Rectifier()
] for _ in range(3)], []),
num_channels=1,
image_size=(1, 1),
tied_biases=False)
cnn.allocate()
assert [
not child.tied_biases for child in cnn.children
if isinstance(child, Convolutional)
]
cnn = ConvolutionalSequence(sum(
[[Convolutional(filter_size=(1, 1), num_filters=1),
Rectifier()] for _ in range(3)], []),
num_channels=1,
image_size=(1, 1),
tied_biases=True)
cnn.allocate()
assert [
child.tied_biases for child in cnn.children
if isinstance(child, Convolutional)
]
def test_untied_biases():
x = tensor.tensor4('x')
num_channels = 4
num_filters = 3
batch_size = 5
filter_size = (3, 3)
conv = Convolutional(filter_size, num_filters, num_channels,
weights_init=Constant(1.), biases_init=Constant(2.),
image_size=(28, 30), tied_biases=False)
conv.initialize()
y = conv.apply(x)
func = function([x], y)
# Untied biases provide a bias for every individual output
assert_allclose(conv.b.eval().shape, (3, 26, 28))
# Untied biases require images of a specific size
x_val_1 = numpy.ones((batch_size, num_channels, 28, 30),
dtype=theano.config.floatX)
assert_allclose(func(x_val_1),
numpy.prod(filter_size) * num_channels *
numpy.ones((batch_size, num_filters, 26, 28)) + 2)
x_val_2 = numpy.ones((batch_size, num_channels, 23, 19),
dtype=theano.config.floatX)
def wrongsize():
func(x_val_2)
assert_raises_regexp(AssertionError, 'AbstractConv shape mismatch',
wrongsize)
def conv_layer(self, name, wt, bias, image_size):
"""Creates a Convolutional brick with the given name, weights,
bias, and image_size."""
layer = Convolutional(
name=name,
filter_size=wt.shape[0:2],
num_channels=wt.shape[2], # in
num_filters=wt.shape[3], # out
weights_init=Constant(0), # does not matter
biases_init=Constant(0), # does not matter
tied_biases=True,
border_mode="valid",
)
if image_size:
layer.image_size = image_size
layer.initialize()
weights = self.to_bc01(wt)
layer.parameters[0].set_value(weights.astype("float32")) # W
layer.parameters[1].set_value(bias.squeeze().astype("float32")) # b
return (layer, layer.get_dim("output")[1:3])
def create_OLD_kim_cnn(layer0_input, embedding_size, input_len, config, pref):
'''
One layer convolution with the same filtersize
'''
filter_width_list = [
int(fw) for fw in config[pref + '_filterwidth'].split()
]
print filter_width_list
num_filters = int(config[pref + '_num_filters'])
totfilters = 0
for i, fw in enumerate(filter_width_list):
num_feature_map = input_len - fw + 1 #39
conv = Convolutional(filter_size=(fw, embedding_size),
num_filters=num_filters,
num_channels=1,
image_size=(input_len, embedding_size),
name="conv" + str(fw))
pooling = MaxPooling((num_feature_map, 1), name="pool" + str(fw))
initialize([conv])
totfilters += num_filters
outpool = Flattener(name="flat" + str(fw)).apply(
Rectifier(name=pref + 'act_' + str(fw)).apply(
pooling.apply(conv.apply(layer0_input))))
if i == 0:
outpools = outpool
else:
outpools = T.concatenate([outpools, outpool], axis=1)
name_rep_len = totfilters
return outpools, name_rep_len
def test_batch_normalization_inside_convolutional_sequence():
"""Test that BN bricks work in ConvolutionalSequences."""
conv_seq = ConvolutionalSequence(
[Convolutional(filter_size=(3, 3), num_filters=4),
BatchNormalization(broadcastable=(False, True, True)),
AveragePooling(pooling_size=(2, 2)),
BatchNormalization(broadcastable=(False, False, False)),
MaxPooling(pooling_size=(2, 2), step=(1, 1))],
weights_init=Constant(1.),
biases_init=Constant(2.),
image_size=(10, 8), num_channels=9)
conv_seq_no_bn = ConvolutionalSequence(
[Convolutional(filter_size=(3, 3), num_filters=4),
AveragePooling(pooling_size=(2, 2)),
MaxPooling(pooling_size=(2, 2), step=(1, 1))],
weights_init=Constant(1.),
biases_init=Constant(2.),
image_size=(10, 8), num_channels=9)
conv_seq.initialize()
conv_seq_no_bn.initialize()
rng = numpy.random.RandomState((2015, 12, 17))
input_ = random_unif(rng, (2, 9, 10, 8))
x = theano.tensor.tensor4()
ybn = conv_seq.apply(x)
y = conv_seq_no_bn.apply(x)
yield (assert_equal, ybn.eval({x: input_}), y.eval({x: input_}))
std = conv_seq.children[-2].population_stdev
std.set_value(3 * std.get_value(borrow=True))
yield (assert_equal, ybn.eval({x: input_}), y.eval({x: input_}) / 3.)
def generate_elementary_block4(self, index):
number_of_channels = 512
name_conv_0 = 'fconv7_' + str(index)
name_relu_0 = 'relu7_' + str(index)
name_conv_3 = 'fconv7_output_' + str(index)
return [Convolutional(filter_size=(1,1), num_filters = 128, border_mode = (0,0), use_bias=True, tied_biases=True, name=name_conv_0, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = number_of_channels), \
ParallelSum2(), Rectifier(name=name_relu_0), \
Convolutional(filter_size=(1,1), num_filters = 1, border_mode = (0,0), use_bias=True, tied_biases=True, name=name_conv_3, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = 128)]
def __init__(self,
filter_size,
num_filters,
num_channels,
batch_size=None,
mid_noise=False,
out_noise=False,
tied_noise=False,
tied_sigma=False,
noise_rate=None,
noise_batch_size=None,
prior_noise_level=None,
image_size=(None, None),
step=(1, 1),
**kwargs):
self.filter_size = filter_size
self.num_filters = num_filters
self.batch_size = batch_size
self.num_channels = num_channels
self.image_size = image_size
self.mid_noise = mid_noise
self.noise_batch_size = noise_batch_size
self.noise_rate = noise_rate
self.step = step
self.border_mode = 'half'
self.tied_biases = True
depth = 2
self.b0 = SpatialBatchNormalization(name='b0')
self.r0 = Rectifier(name='r0')
self.n0 = (SpatialNoise(name='n0',
noise_rate=self.noise_rate,
tied_noise=tied_noise,
tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level)
if mid_noise else None)
self.c0 = Convolutional(name='c0')
self.b1 = SpatialBatchNormalization(name='b1')
self.r1 = Rectifier(name='r1')
self.n1 = (SpatialNoise(name='n1',
noise_rate=self.noise_rate,
tied_noise=tied_noise,
tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level)
if out_noise else None)
self.c1 = Convolutional(name='c1')
kwargs.setdefault('children', []).extend([
c for c in [
self.c0, self.b0, self.r0, self.n0, self.c1, self.b1, self.r1,
self.n1
] if c is not None
])
super(ResidualConvolutional, self).__init__(**kwargs)
def __init__(self, image_dimension, **kwargs):
layers = []
#############################################
# a first block with 2 convolutions of 32 (3, 3) filters
layers.append(Convolutional((3, 3), 32, border_mode='half'))
layers.append(Rectifier())
layers.append(Convolutional((3, 3), 32, border_mode='half'))
layers.append(Rectifier())
# maxpool with size=(2, 2)
layers.append(MaxPooling((2, 2)))
#############################################
# a 2nd block with 3 convolutions of 64 (3, 3) filters
layers.append(Convolutional((3, 3), 64, border_mode='half'))
layers.append(Rectifier())
layers.append(Convolutional((3, 3), 64, border_mode='half'))
layers.append(Rectifier())
layers.append(Convolutional((3, 3), 64, border_mode='half'))
layers.append(Rectifier())
# maxpool with size=(2, 2)
layers.append(MaxPooling((2, 2)))
#############################################
# a 3rd block with 4 convolutions of 128 (3, 3) filters
layers.append(Convolutional((3, 3), 128, border_mode='half'))
layers.append(Rectifier())
layers.append(Convolutional((3, 3), 128, border_mode='half'))
layers.append(Rectifier())
layers.append(Convolutional((3, 3), 128, border_mode='half'))
layers.append(Rectifier())
layers.append(Convolutional((3, 3), 128, border_mode='half'))
layers.append(Rectifier())
# maxpool with size=(2, 2)
layers.append(MaxPooling((2, 2)))
self.conv_sequence = ConvolutionalSequence(layers,
3,
image_size=image_dimension)
flattener = Flattener()
self.top_mlp = MLP(activations=[Rectifier(), Logistic()],
dims=[500, 1])
application_methods = [
self.conv_sequence.apply, flattener.apply, self.top_mlp.apply
]
super(VGGNet, self).__init__(application_methods,
biases_init=Constant(0),
weights_init=Uniform(width=.1),
**kwargs)
class ConvolutionalActivation(Initializable):
"""A convolution followed by an activation function.
Parameters
----------
activation : :class:`.BoundApplication`
The application method to apply after convolution (i.e.
the nonlinear activation function)
See Also
--------
:class:`Convolutional` : For the documentation of other parameters.
"""
@lazy(allocation=['filter_size', 'num_filters', 'num_channels'])
def __init__(self, activation, filter_size, num_filters, num_channels,
batch_size=None, image_size=None, step=(1, 1),
border_mode='valid', tied_biases=False, **kwargs):
self.convolution = Convolutional(name='conv'+ kwargs['name'])
self.bn = BatchNorm(name='bn'+ kwargs['name'])
self.activation = activation
self.filter_size = filter_size
self.num_filters = num_filters
self.num_channels = num_channels
self.batch_size = batch_size
self.image_size = image_size
self.step = step
self.border_mode = border_mode
self.tied_biases = tied_biases
super(ConvolutionalActivation, self).__init__(**kwargs)
self.children = [self.convolution, self.bn, self.activation]
def _push_allocation_config(self):
for attr in ['filter_size', 'num_filters', 'step', 'border_mode',
'batch_size', 'num_channels', 'image_size',
'tied_biases']:
setattr(self.convolution, attr, getattr(self, attr))
setattr(self.bn, 'input_dim', self.num_filters)
def get_dim(self, name):
# TODO The name of the activation output doesn't need to be `output`
return self.convolution.get_dim(name)
def apply(self, input_):
out = self.convolution.apply(input_)
out = self.bn.apply(out)
out = self.activation.apply(out)
return out
def inference(self, input_):
out = self.convolution.apply(input_)
out = self.bn.inference(out)
out = self.activation.apply(out)
return out
def test_convolutional_sequence_with_convolutions_raw_activation():
seq = ConvolutionalSequence(
[Convolutional(filter_size=(3, 3), num_filters=4),
Rectifier(),
Convolutional(filter_size=(5, 5), num_filters=3, step=(2, 2)),
Tanh()],
num_channels=2,
image_size=(21, 39))
seq.allocate()
x = theano.tensor.tensor4()
out = seq.apply(x).eval({x: numpy.ones((10, 2, 21, 39),
dtype=theano.config.floatX)})
assert out.shape == (10, 3, 8, 17)
def test_convolutional():
x = tensor.tensor4('x')
num_channels = 4
num_filters = 3
batch_size = 5
filter_size = (3, 3)
conv = Convolutional(filter_size,
num_filters,
num_channels,
image_size=(17, 13),
weights_init=Constant(1.),
biases_init=Constant(5.))
conv.initialize()
y = conv.apply(x)
func = function([x], y)
x_val = numpy.ones((batch_size, num_channels, 17, 13),
dtype=theano.config.floatX)
assert_allclose(
func(x_val),
numpy.prod(filter_size) * num_channels * numpy.ones(
(batch_size, num_filters, 15, 11)) + 5)
conv.image_size = (17, 13)
conv.batch_size = 2 # This should have effect on get_dim
assert conv.get_dim('output') == (num_filters, 15, 11)
def generate_elementary_block1(self, index, to_index):
number_of_channels = 512
name_conv_0 = 'fconv7_' + str(index)
name_relu_0 = 'relu7_' + str(index)
name_conv_1 = 'fconv7_' + str(index) + 'to' + str(to_index) + '_step1'
name_relu_1 = 'relu7_' + str(index) + 'to' + str(to_index) + '_step1'
name_conv_2 = 'fconv7_' + str(index) + 'to' + str(to_index) + '_step2'
name_conv_3 = 'fconv7_output_' + str(index)
return [Convolutional(filter_size=(1,1), num_filters = 128, border_mode = (0,0), use_bias=True, tied_biases=True, name=name_conv_0, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = number_of_channels), \
Rectifier(name=name_relu_0), \
Convolutional(filter_size=(7,7), num_filters = 64, border_mode = (3,3), use_bias=True, tied_biases=True, name=name_conv_1, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = 128), \
Rectifier(name=name_relu_1), \
Convolutional(filter_size=(7,7), num_filters = 128, border_mode = (3,3), use_bias=True, tied_biases=True, name=name_conv_2, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = 64), \
Convolutional(filter_size=(1,1), num_filters = 1, border_mode = (0,0), use_bias=True, tied_biases=True, name=name_conv_3, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = 128)]
def test_no_input_size():
# suppose x is outputted by some RNN
x = tensor.tensor4('x')
filter_size = (1, 3)
num_filters = 2
num_channels = 5
c = Convolutional(filter_size,
num_filters,
num_channels,
tied_biases=True,
weights_init=Constant(1.),
biases_init=Constant(1.))
c.initialize()
out = c.apply(x)
assert c.get_dim('output') == (2, None, None)
assert out.ndim == 4
c = Convolutional(filter_size,
num_filters,
num_channels,
tied_biases=False,
weights_init=Constant(1.),
biases_init=Constant(1.))
assert_raises_regexp(ValueError, 'Cannot infer bias size \S+',
c.initialize)
def conv_block(input_img,
n_filter,
filter_size,
input_featuremap_size,
ordering=''):
# found in torch spatialconvolution
std0 = 2. / (filter_size[0] * filter_size[1] *
input_featuremap_size[0])**.5
std1 = 2. / (input_featuremap_size[0])**.5
layers = []
layers.append(
Convolutional(filter_size=filter_size,
num_filters=n_filter,
border_mode='half',
name='conv%s_1' % (ordering, ),
use_bias=True,
weights_init=Uniform(width=std0)))
layers.append(BatchNormalization(name='bn%s_1' % (ordering, )))
layers.append(LeakyReLU())
layers.append(
Convolutional(filter_size=filter_size,
num_filters=n_filter,
border_mode='half',
name='conv%s_2' % (ordering, ),
use_bias=True,
weights_init=Uniform(width=std0)))
layers.append(BatchNormalization(name='bn%s_2' % (ordering, )))
layers.append(LeakyReLU())
layers.append(
Convolutional(filter_size=(1, 1),
num_filters=n_filter,
border_mode='valid',
name='conv%s_3b' % (ordering, ),
use_bias=True,
weights_init=Uniform(width=std1)))
layers.append(BatchNormalization(name='bn%s_3' % (ordering, )))
layers.append(LeakyReLU())
conv_sequence = ConvolutionalSequence(
layers,
num_channels=input_featuremap_size[0],
image_size=(input_featuremap_size[1], input_featuremap_size[2]),
biases_init=Uniform(width=.1),
name='convsequence%s' % (ordering, ))
conv_sequence.initialize()
return conv_sequence.apply(input_img)
def __init__(self, activation, filter_size, num_filters, num_channels,
batch_size=None, image_size=None, step=(1, 1),
border_mode='valid', tied_biases=False, **kwargs):
self.convolution = Convolutional(name='conv'+ kwargs['name'])
self.bn = BatchNorm(name='bn'+ kwargs['name'])
self.activation = activation
self.filter_size = filter_size
self.num_filters = num_filters
self.num_channels = num_channels
self.batch_size = batch_size
self.image_size = image_size
self.step = step
self.border_mode = border_mode
self.tied_biases = tied_biases
super(ConvolutionalActivation, self).__init__(**kwargs)
self.children = [self.convolution, self.bn, self.activation]
def conv_brick(filter_size, step, num_filters, border_mode='valid'):
"""Instantiates a ConvolutionalBrick."""
return Convolutional(filter_size=(filter_size, filter_size),
step=(step, step),
border_mode=border_mode,
num_filters=num_filters,
name=name_generator())
def __init__(self,
conv_activations,
num_channels,
image_shape,
filter_sizes,
feature_maps,
pooling_sizes,
top_mlp_activations,
top_mlp_dims,
conv_step=None,
border_mode='valid',
**kwargs):
if conv_step is None:
self.conv_step = (1, 1)
else:
self.conv_step = conv_step
self.num_channels = num_channels
self.image_shape = image_shape
self.top_mlp_activations = top_mlp_activations
self.top_mlp_dims = top_mlp_dims
self.border_mode = border_mode
conv_parameters = zip(filter_sizes, feature_maps)
# Construct convolutional, activation, and pooling layers with corresponding parameters
self.convolution_layer = (
Convolutional(filter_size=filter_size,
num_filters=num_filter,
step=self.conv_step,
border_mode=self.border_mode,
name='conv_{}'.format(i))
for i, (filter_size, num_filter) in enumerate(conv_parameters))
self.BN_layer = (BatchNormalization(name='bn_conv_{}'.format(i))
for i in enumerate(conv_parameters))
self.pooling_layer = (MaxPooling(size, name='pool_{}'.format(i))
for i, size in enumerate(pooling_sizes))
self.layers = list(
interleave([
self.convolution_layer, self.BN_layer, conv_activations,
self.pooling_layer
]))
self.conv_sequence = ConvolutionalSequence(self.layers,
num_channels,
image_size=image_shape)
# Construct a top MLP
self.top_mlp = MLP(top_mlp_activations, top_mlp_dims)
# We need to flatten the output of the last convolutional layer.
# This brick accepts a tensor of dimension (batch_size, ...) and
# returns a matrix (batch_size, features)
self.flattener = Flattener()
application_methods = [
self.conv_sequence.apply, self.flattener.apply, self.top_mlp.apply
]
super(LeNet, self).__init__(application_methods, **kwargs)
def test_convolutional_sequence_with_no_input_size():
# suppose x is outputted by some RNN
x = tensor.tensor4('x')
filter_size = (1, 1)
num_filters = 2
num_channels = 1
pooling_size = (1, 1)
conv = Convolutional(filter_size,
num_filters,
tied_biases=False,
weights_init=Constant(1.),
biases_init=Constant(1.))
act = Rectifier()
pool = MaxPooling(pooling_size)
bad_seq = ConvolutionalSequence([conv, act, pool],
num_channels,
tied_biases=False)
assert_raises_regexp(ValueError, 'Cannot infer bias size \S+',
bad_seq.initialize)
seq = ConvolutionalSequence([conv, act, pool],
num_channels,
tied_biases=True)
try:
seq.initialize()
out = seq.apply(x)
except TypeError:
assert False, "This should have succeeded"
assert out.ndim == 4
def __init__(self, **kwargs):
children = []
self.list_simple_joints = [19, 20, 8, 7] + range(16,19)[::-1] + range(4, 7)[::-1] + [1] + [15, 3, 2] + [0]
self.simple_joints = {}
for simple_joint_idx in self.list_simple_joints:
self.simple_joints[simple_joint_idx] = [Convolutional(filter_size=(1,1), num_filters = 512, border_mode = (0,0), use_bias=True, tied_biases=True, name='fconv_' + str(simple_joint_idx), biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = 512), \
Rectifier(name='fconv_relu' + str(simple_joint_idx)), \
Convolutional(filter_size=(1,1), num_filters = 1, border_mode = (0,0), use_bias=True, tied_biases=True, name='fconv1_' + str(simple_joint_idx), biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = 512), \
Logistic(name = 'flogistic_' + str(simple_joint_idx))]
children += self.simple_joints[simple_joint_idx]
kwargs.setdefault('children', []).extend(children)
super(top_direction_block, self).__init__(**kwargs)
def build_pipeline(self, input_shape, params):
from blocks.bricks import Tanh, Sequence
from blocks.bricks.conv import Convolutional, MaxPooling
from blocks.initialization import Uniform
_, num_channels, input_len, num_freqs = input_shape # bc01
# Note: this layer is linear
conv = Convolutional(
name='conv',
filter_size=(params['filter_width_time'],
params['filter_width_freq']),
num_filters=params['num_components'], # out
num_channels=num_channels, # in
image_size=(input_len, num_freqs),
weights_init=Uniform(mean=0, std=0.01),
use_bias=params['use_bias'])
tanh = Tanh()
# optional pooling
if params['pool_width_time'] > 1 or params['pool_width_freq'] > 1:
pool = MaxPooling(
(params['pool_width_time'], params['pool_width_freq']),
step=(params['pool_stride_time'], params['pool_stride_freq']))
pipeline = Sequence([conv.apply, tanh.apply, pool.apply],
name='pipeline')
else:
pipeline = Sequence([conv.apply, tanh.apply], name='pipeline')
pipeline.initialize()
return pipeline
def create_OLD_kim_cnn(layer0_input, embedding_size, input_len, config, pref):
'''
One layer convolution with the same filtersize
'''
filter_width_list = [int(fw) for fw in config[pref + '_filterwidth'].split()]
print filter_width_list
num_filters = int(config[pref+'_num_filters'])
totfilters = 0
for i, fw in enumerate(filter_width_list):
num_feature_map = input_len - fw + 1 #39
conv = Convolutional(
filter_size=(fw, embedding_size),
num_filters=num_filters,
num_channels=1,
image_size=(input_len, embedding_size),
name="conv" + str(fw))
pooling = MaxPooling((num_feature_map,1), name="pool"+str(fw))
initialize([conv])
totfilters += num_filters
outpool = Flattener(name="flat"+str(fw)).apply(Rectifier(name=pref+'act_'+str(fw)).apply(pooling.apply(conv.apply(layer0_input))))
if i == 0:
outpools = outpool
else:
outpools = T.concatenate([outpools, outpool], axis=1)
name_rep_len = totfilters
return outpools, name_rep_len
def test_convolutional():
x = tensor.tensor4("x")
num_channels = 4
num_filters = 3
batch_size = 5
filter_size = (3, 3)
conv = Convolutional(
filter_size,
num_filters,
num_channels,
image_size=(17, 13),
weights_init=Constant(1.0),
biases_init=Constant(5.0),
)
conv.initialize()
y = conv.apply(x)
func = function([x], y)
x_val = numpy.ones((batch_size, num_channels, 17, 13), dtype=theano.config.floatX)
assert_allclose(
func(x_val), numpy.prod(filter_size) * num_channels * numpy.ones((batch_size, num_filters, 15, 11)) + 5
)
conv.image_size = (17, 13)
conv.batch_size = 2 # This should have effect on get_dim
assert conv.get_dim("output") == (num_filters, 15, 11)
def __init__(self, filter_size, num_filters, num_channels, noise_batch_size,
image_size=(None, None), step=(1, 1), border_mode='valid',
tied_biases=True,
prior_mean=0, prior_noise_level=0, **kwargs):
self.convolution = Convolutional()
self.mask = Convolutional(name='mask')
children = [self.convolution, self.mask]
kwargs.setdefault('children', []).extend(children)
super(NoisyConvolutional, self).__init__(**kwargs)
self.filter_size = filter_size
self.num_filters = num_filters
self.num_channels = num_channels
self.noise_batch_size = noise_batch_size
self.image_size = image_size
self.step = step
self.border_mode = border_mode
self.tied_biases = tied_biases
self.prior_mean = prior_mean
self.prior_noise_level = prior_noise_level
def test_fully_layer():
batch_size=2
x = T.tensor4();
y = T.ivector()
V = 200
layer_conv = Convolutional(filter_size=(5,5),num_filters=V,
name="toto",
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0.0))
# try with no bias
activation = Rectifier()
pool = MaxPooling(pooling_size=(2,2))
convnet = ConvolutionalSequence([layer_conv, activation, pool], num_channels=15,
image_size=(10,10),
name="conv_section")
convnet.push_allocation_config()
convnet.initialize()
output=convnet.apply(x)
batch_size=output.shape[0]
output_dim=np.prod(convnet.get_dim('output'))
result_conv = output.reshape((batch_size, output_dim))
mlp=MLP(activations=[Rectifier().apply], dims=[output_dim, 10],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0.0))
mlp.initialize()
output=mlp.apply(result_conv)
cost = T.mean(Softmax().categorical_cross_entropy(y.flatten(), output))
cg = ComputationGraph(cost)
W = VariableFilter(roles=[WEIGHT])(cg.variables)
B = VariableFilter(roles=[BIAS])(cg.variables)
W = W[0]; b = B[0]
inputs_fully = VariableFilter(roles=[INPUT], bricks=[Linear])(cg)
outputs_fully = VariableFilter(roles=[OUTPUT], bricks=[Linear])(cg)
var_input=inputs_fully[0]
var_output=outputs_fully[0]
[d_W,d_S,d_b] = T.grad(cost, [W, var_output, b])
d_b = d_b.dimshuffle(('x',0))
d_p = T.concatenate([d_W, d_b], axis=0)
x_value = 1e3*np.random.ranf((2,15, 10, 10))
f = theano.function([x,y], [var_input, d_S, d_p], allow_input_downcast=True, on_unused_input='ignore')
A, B, C= f(x_value, [5, 0])
A = np.concatenate([A, np.ones((2,1))], axis=1)
print 'A', A.shape
print 'B', B.shape
print 'C', C.shape
print lin.norm(C - np.dot(np.transpose(A), B), 'fro')
return
"""
def test_convolutional_sequence_activation_get_dim():
seq = ConvolutionalSequence([Tanh()], num_channels=9, image_size=(4, 6))
seq.allocate()
assert seq.get_dim('output') == (9, 4, 6)
seq = ConvolutionalSequence([Convolutional(filter_size=(7, 7),
num_filters=5,
border_mode=(1, 1)),
Tanh()], num_channels=8, image_size=(8, 11))
seq.allocate()
assert seq.get_dim('output') == (5, 4, 7)
def test_convolutional_sequence_use_bias():
cnn = ConvolutionalSequence(
sum([[Convolutional(filter_size=(1, 1), num_filters=1), Rectifier()]
for _ in range(3)], []),
num_channels=1, image_size=(1, 1),
use_bias=False)
cnn.allocate()
x = tensor.tensor4()
y = cnn.apply(x)
params = ComputationGraph(y).parameters
assert len(params) == 3 and all(param.name == 'W' for param in params)
def test_border_mode_not_pushed():
layers = [Convolutional(border_mode='full'),
Convolutional(),
Rectifier(),
Convolutional(border_mode='valid'),
Rectifier(),
Convolutional(border_mode='full'),
Rectifier()]
stack = ConvolutionalSequence(layers)
stack.push_allocation_config()
assert stack.children[0].border_mode == 'full'
assert stack.children[1].border_mode == 'valid'
assert stack.children[3].border_mode == 'valid'
assert stack.children[5].border_mode == 'full'
stack2 = ConvolutionalSequence(layers, border_mode='full')
stack2.push_allocation_config()
assert stack2.children[0].border_mode == 'full'
assert stack2.children[1].border_mode == 'full'
assert stack2.children[3].border_mode == 'full'
assert stack2.children[5].border_mode == 'full'
def create_yy_cnn(numConvLayer, conv_input, embedding_size, input_len, config,
pref):
'''
CNN with several layers of convolution, each with specific filter size.
Maxpooling at the end.
'''
filter_width_list = [
int(fw) for fw in config[pref + '_filterwidth'].split()
]
base_num_filters = int(config[pref + '_num_filters'])
assert len(filter_width_list) == numConvLayer
convs = []
fmlist = []
last_fm = input_len
for i in range(numConvLayer):
fw = filter_width_list[i]
num_feature_map = last_fm - fw + 1 #39
conv = Convolutional(image_size=(last_fm, embedding_size),
filter_size=(fw, embedding_size),
num_filters=min(int(config[pref + '_maxfilter']),
base_num_filters * fw),
num_channels=1)
fmlist.append(num_feature_map)
last_fm = num_feature_map
embedding_size = conv.num_filters
convs.append(conv)
initialize(convs)
for i, conv in enumerate(convs):
conv.name = pref + '_conv' + str(i)
conv_input = conv.apply(conv_input)
conv_input = conv_input.flatten().reshape(
(conv_input.shape[0], 1, fmlist[i], conv.num_filters))
lastconv = conv
lastconv_out = conv_input
pool_layer = MaxPooling(pooling_size=(last_fm, 1))
pool_layer.name = pref + '_pool_' + str(fw)
act = Rectifier()
act.name = 'act_' + str(fw)
outpool = act.apply(pool_layer.apply(lastconv_out).flatten(2))
return outpool, lastconv.num_filters
def build_model(Zs):
# first block
z1 = conv_block(Zs['Z1'], 8, (3, 3), (3, 256, 256), '1')
z2 = conv_block(Zs['Z2'], 8, (3, 3), (3, 128, 128), '2')
z3 = conv_block(Zs['Z3'], 8, (3, 3), (3, 64, 64), '3')
z4 = conv_block(Zs['Z4'], 8, (3, 3), (3, 32, 32), '4')
z5 = conv_block(Zs['Z5'], 8, (3, 3), (3, 16, 16), '5')
# merge 4 & 5
m45 = join(z5, z4, 8, 8, (32, 32), '45')
# merge 3 & 45
c45 = conv_block(m45, 16, (3, 3), (16, 32, 32), '45')
m345 = join(c45, z3, 16, 8, (64, 64))
# merge 2 & 345
c345 = conv_block(m345, 24, (3, 3), (24, 64, 64), '345')
m2345 = join(c345, z2, 24, 8, (128, 128), '345')
# merge 1 & 2345
c2345 = conv_block(m2345, 32, (3, 3), (32, 128, 128), '2345')
m12345 = join(c2345, z1, 32, 8, (256, 256), '2345')
last_conv_block = conv_block(m12345, 40, (3, 3), (40, 256, 256), '12345')
conv_out = Convolutional((1, 1),
3,
40,
image_size=(256, 256),
biases_init=Uniform(width=.1),
weights_init=Uniform(width=.3),
use_bias=True,
name='conv_out')
conv_out.initialize()
conv_out_output = conv_out.apply(last_conv_block)
mean = tensor.addbroadcast(
theano.shared(MEAN_VALUES).astype('float32'), 0, 2, 3)
return 128. * (1 + conv_out_output) - mean
def __init__(self, filter_size, num_filters, num_channels,
batch_size=None,
mid_noise=False,
out_noise=False,
tied_noise=False,
tied_sigma=False,
noise_rate=None,
noise_batch_size=None,
prior_noise_level=None,
image_size=(None, None), step=(1, 1),
**kwargs):
self.filter_size = filter_size
self.num_filters = num_filters
self.batch_size = batch_size
self.num_channels = num_channels
self.image_size = image_size
self.mid_noise = mid_noise
self.noise_batch_size = noise_batch_size
self.noise_rate = noise_rate
self.step = step
self.border_mode = 'half'
self.tied_biases = True
depth = 2
self.b0 = SpatialBatchNormalization(name='b0')
self.r0 = Rectifier(name='r0')
self.n0 = (SpatialNoise(name='n0', noise_rate=self.noise_rate,
tied_noise=tied_noise, tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level) if mid_noise else None)
self.c0 = Convolutional(name='c0')
self.b1 = SpatialBatchNormalization(name='b1')
self.r1 = Rectifier(name='r1')
self.n1 = (SpatialNoise(name='n1', noise_rate=self.noise_rate,
tied_noise=tied_noise, tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level) if out_noise else None)
self.c1 = Convolutional(name='c1')
kwargs.setdefault('children', []).extend([c for c in [
self.c0, self.b0, self.r0, self.n0,
self.c1, self.b1, self.r1, self.n1] if c is not None])
super(ResidualConvolutional, self).__init__(**kwargs)
def create_yy_cnn(numConvLayer, conv_input, embedding_size, input_len, config, pref):
'''
CNN with several layers of convolution, each with specific filter size.
Maxpooling at the end.
'''
filter_width_list = [int(fw) for fw in config[pref + '_filterwidth'].split()]
base_num_filters = int(config[pref + '_num_filters'])
assert len(filter_width_list) == numConvLayer
convs = []; fmlist = []
last_fm = input_len
for i in range(numConvLayer):
fw = filter_width_list[i]
num_feature_map = last_fm - fw + 1 #39
conv = Convolutional(
image_size=(last_fm, embedding_size),
filter_size=(fw, embedding_size),
num_filters=min(int(config[pref + '_maxfilter']), base_num_filters * fw),
num_channels=1
)
fmlist.append(num_feature_map)
last_fm = num_feature_map
embedding_size = conv.num_filters
convs.append(conv)
initialize(convs)
for i, conv in enumerate(convs):
conv.name = pref+'_conv' + str(i)
conv_input = conv.apply(conv_input)
conv_input = conv_input.flatten().reshape((conv_input.shape[0], 1, fmlist[i], conv.num_filters))
lastconv = conv
lastconv_out = conv_input
pool_layer = MaxPooling(
pooling_size=(last_fm,1)
)
pool_layer.name = pref+'_pool_' + str(fw)
act = Rectifier(); act.name = 'act_' + str(fw)
outpool = act.apply(pool_layer.apply(lastconv_out).flatten(2))
return outpool, lastconv.num_filters
def __init__(self, input_dim, noise_batch_size=None, noise_rate=None,
tied_noise=False, tied_sigma=False,
prior_mean=0, prior_noise_level=0, **kwargs):
self.mask = Convolutional(name='mask')
self.flatten = GlobalAverageFlattener() if tied_sigma else None
children = list(p for p in [self.mask, self.flatten] if p is not None)
kwargs.setdefault('children', []).extend(children)
super(SpatialNoise, self).__init__(**kwargs)
self.input_dim = input_dim
self.tied_noise = tied_noise
self.tied_sigma = tied_sigma
self.noise_batch_size = noise_batch_size
self.noise_rate = noise_rate if noise_rate is not None else 1.0
self.prior_mean = prior_mean
self.prior_noise_level = prior_noise_level
self._training_mode = []
def __init__(self, activation, filter_size, num_filters, num_channels,
batch_size=None, image_size=None, step=(1, 1),
border_mode='valid', **kwargs):
self.convolution = Convolutional()
self.filter_size = filter_size
self.num_filters = num_filters
self.num_channels = num_channels
self.batch_size = batch_size
self.image_size = image_size
self.step = step
self.border_mode = border_mode
super(ConvolutionalActivation, self).__init__(
application_methods=[self.convolution.apply, activation],
**kwargs)
class ConvolutionalActivation(Sequence, Initializable):
"""A convolution followed by an activation function.
Parameters
----------
activation : :class:`.BoundApplication`
The application method to apply after convolution (i.e.
the nonlinear activation function)
See Also
--------
:class:`Convolutional` for the other parameters.
"""
@lazy(allocation=['filter_size', 'num_filters', 'num_channels'])
def __init__(self, activation, filter_size, num_filters, num_channels,
batch_size=None, image_size=None, step=(1, 1),
border_mode='valid', **kwargs):
self.convolution = Convolutional()
self.filter_size = filter_size
self.num_filters = num_filters
self.num_channels = num_channels
self.batch_size = batch_size
self.image_size = image_size
self.step = step
self.border_mode = border_mode
super(ConvolutionalActivation, self).__init__(
application_methods=[self.convolution.apply, activation],
**kwargs)
def _push_allocation_config(self):
for attr in ['filter_size', 'num_filters', 'step', 'border_mode',
'batch_size', 'num_channels', 'image_size']:
setattr(self.convolution, attr, getattr(self, attr))
def get_dim(self, name):
# TODO The name of the activation output doesn't need to be `output`
return self.convolution.get_dim(name)
def test_no_input_size():
# suppose x is outputted by some RNN
x = tensor.tensor4('x')
filter_size = (1, 3)
num_filters = 2
num_channels = 5
c = Convolutional(filter_size, num_filters, num_channels, tied_biases=True,
weights_init=Constant(1.), biases_init=Constant(1.))
c.initialize()
out = c.apply(x)
assert c.get_dim('output') == (2, None, None)
assert out.ndim == 4
c = Convolutional(filter_size, num_filters, num_channels,
tied_biases=False, weights_init=Constant(1.),
biases_init=Constant(1.))
assert_raises_regexp(ValueError, 'Cannot infer bias size \S+',
c.initialize)
class NoisyConvolutional(Initializable, Feedforward, Random):
"""Convolutional transformation sent through a learned noisy channel.
Parameters (same as Convolutional)
"""
@lazy(allocation=[
'filter_size', 'num_filters', 'num_channels', 'noise_batch_size'])
def __init__(self, filter_size, num_filters, num_channels, noise_batch_size,
image_size=(None, None), step=(1, 1), border_mode='valid',
tied_biases=True,
prior_mean=0, prior_noise_level=0, **kwargs):
self.convolution = Convolutional()
self.mask = Convolutional(name='mask')
children = [self.convolution, self.mask]
kwargs.setdefault('children', []).extend(children)
super(NoisyConvolutional, self).__init__(**kwargs)
self.filter_size = filter_size
self.num_filters = num_filters
self.num_channels = num_channels
self.noise_batch_size = noise_batch_size
self.image_size = image_size
self.step = step
self.border_mode = border_mode
self.tied_biases = tied_biases
self.prior_mean = prior_mean
self.prior_noise_level = prior_noise_level
def _push_allocation_config(self):
self.convolution.filter_size = self.filter_size
self.convolution.num_filters = self.num_filters
self.convolution.num_channels = self.num_channels
# self.convolution.batch_size = self.batch_size
self.convolution.image_size = self.image_size
self.convolution.step = self.step
self.convolution.border_mode = self.border_mode
self.convolution.tied_biases = self.tied_biases
self.mask.filter_size = (1, 1)
self.mask.num_filters = self.num_filters
self.mask.num_channels = self.num_filters
# self.mask.batch_size = self.batch_size
self.mask.image_size = self.convolution.get_dim('output')[1:]
# self.mask.step = self.step
# self.mask.border_mode = self.border_mode
self.mask.tied_biases = self.tied_biases
def _allocate(self):
out_shape = self.convolution.get_dim('output')
N = shared_floatx_zeros((self.noise_batch_size,) + out_shape, name='N')
add_role(N, NOISE)
self.parameters.append(N)
@application(inputs=['input_'], outputs=['output'])
def apply(self, input_, application_call):
"""Apply the linear transformation followed by masking with noise.
Parameters
----------
input_ : :class:`~tensor.TensorVariable`
The input on which to apply the transformations
Returns
-------
output : :class:`~tensor.TensorVariable`
The transformed input
"""
from theano.printing import Print
pre_noise = self.convolution.apply(input_)
# noise_level = self.mask.apply(input_)
noise_level = (self.prior_noise_level -
tensor.clip(self.mask.apply(pre_noise), -16, 16))
noise_level = copy_and_tag_noise(
noise_level, self, LOG_SIGMA, 'log_sigma')
# Allow incomplete batches by just taking the noise that is needed
noise = self.parameters[0][:noise_level.shape[0], :, :, :]
# noise = self.theano_rng.normal(noise_level.shape)
kl = (
self.prior_noise_level - noise_level
+ 0.5 * (
tensor.exp(2 * noise_level)
+ (pre_noise - self.prior_mean) ** 2
) / tensor.exp(2 * self.prior_noise_level)
- 0.5
)
application_call.add_auxiliary_variable(kl, roles=[NITS], name='nits')
return pre_noise + tensor.exp(noise_level) * noise
def get_dim(self, name):
if name == 'input_':
return self.convolution.get_dim(name)
if name == 'output':
return self.convolution.get_dim(name)
if name == 'nits':
return self.convolution.get_dim('output')
return super(NoisyConvolutional, self).get_dim(name)
@property
def num_output_channels(self):
return self.num_filters
class SpatialNoise(NoiseLayer, Initializable, Random):
"""A learned noise layer.
"""
@lazy(allocation=['input_dim'])
def __init__(self, input_dim, noise_batch_size=None, noise_rate=None,
tied_noise=False, tied_sigma=False,
prior_mean=0, prior_noise_level=0, **kwargs):
self.mask = Convolutional(name='mask')
self.flatten = GlobalAverageFlattener() if tied_sigma else None
children = list(p for p in [self.mask, self.flatten] if p is not None)
kwargs.setdefault('children', []).extend(children)
super(SpatialNoise, self).__init__(**kwargs)
self.input_dim = input_dim
self.tied_noise = tied_noise
self.tied_sigma = tied_sigma
self.noise_batch_size = noise_batch_size
self.noise_rate = noise_rate if noise_rate is not None else 1.0
self.prior_mean = prior_mean
self.prior_noise_level = prior_noise_level
self._training_mode = []
def _push_allocation_config(self):
self.mask.filter_size = (1, 1)
self.mask.num_filters = self.num_channels
self.mask.num_channels = self.num_channels
self.mask.image_size = self.image_size
def _allocate(self):
if self.noise_batch_size is not None:
if self.tied_noise:
N = shared_floatx_zeros(
(self.noise_batch_size, self.input_dim[0]), name='N')
else:
N = shared_floatx_zeros(
(self.noise_batch_size,) + self.input_dim, name='N')
add_role(N, NOISE)
self.parameters.append(N)
@application(inputs=['input_'], outputs=['output'])
def apply(self, input_, application_call):
"""Apply the linear transformation followed by masking with noise.
Parameters
----------
input_ : :class:`~tensor.TensorVariable`
The input on which to apply the transformations
Returns
-------
output : :class:`~tensor.TensorVariable`
The transformed input
"""
# When not in training mode, turn off noise
if not self._training_mode:
return input_
if self.tied_sigma:
average = tensor.shape_padright(self.flatten.apply(input_), 2)
noise_level = (self.prior_noise_level -
tensor.clip(self.mask.apply(average), -16, 16))
noise_level = tensor.patternbroadcast(noise_level,
(False, False, True, True))
noise_level = copy_and_tag_noise(
noise_level, self, LOG_SIGMA, 'log_sigma')
else:
average = input_
noise_level = (self.prior_noise_level -
tensor.clip(self.mask.apply(input_), -16, 16))
noise_level = copy_and_tag_noise(
noise_level, self, LOG_SIGMA, 'log_sigma')
# Allow incomplete batches by just taking the noise that is needed
if self.tied_noise:
if self.noise_batch_size is not None:
noise = self.parameters[0][:input_.shape[0], :]
else:
noise = self.theano_rng.normal(input_.shape[0:2])
noise = tensor.shape_padright(2)
else:
if self.noise_batch_size is not None:
noise = self.parameters[0][:input_.shape[0], :, :, :]
else:
noise = self.theano_rng.normal(input_.shape)
kl = (
self.prior_noise_level - noise_level
+ 0.5 * (
tensor.exp(2 * noise_level)
+ (average - self.prior_mean) ** 2
) / tensor.exp(2 * self.prior_noise_level)
- 0.5
)
application_call.add_auxiliary_variable(kl, roles=[NITS], name='nits')
return input_ + self.noise_rate * tensor.exp(noise_level) * noise
# Needed for the Feedforward interface.
@property
def output_dim(self):
return self.input_dim
# The following properties allow for BatchNormalization bricks
# to be used directly inside of a ConvolutionalSequence.
@property
def image_size(self):
return self.input_dim[-2:]
@image_size.setter
def image_size(self, value):
if not isinstance(self.input_dim, collections.Sequence):
self.input_dim = (None,) + tuple(value)
else:
self.input_dim = (self.input_dim[0],) + tuple(value)
@property
def num_channels(self):
return self.input_dim[0]
@num_channels.setter
def num_channels(self, value):
if not isinstance(self.input_dim, collections.Sequence):
self.input_dim = (value,) + (None, None)
else:
self.input_dim = (value,) + self.input_dim[-2:]
def get_dim(self, name):
if name in ('input', 'output'):
return self.input_dim
else:
raise KeyError
@property
def num_output_channels(self):
return self.num_channels
from theano import tensor
x = tensor.matrix('features')
from blocks.bricks import Linear, Rectifier, Softmax
from blocks.bricks.conv import Convolutional, ConvolutionalActivation
input_to_hidden = Convolutional((5,5), 32, 1,border_mode='same')
h = Rectifier().apply(input_to_hidden.apply(x))
hidden_to_output = Linear(name='hidden_to_output', input_dim=100, output_dim=10)
y_hat = Softmax().apply(hidden_to_output.apply(h))
y = tensor.lmatrix('targets')
from blocks.bricks.cost import CategoricalCrossEntropy
cost = CategoricalCrossEntropy().apply(y.flatten(), y_hat)
from blocks.bricks import WEIGHT
from blocks.graph import ComputationGraph
from blocks.filter import VariableFilter
cg = ComputationGraph(cost)
W1, W2 = VariableFilter(roles=[WEIGHT])(cg.variables)
cost = cost + 0.005 * (W1 ** 2).sum() + 0.005 * (W2 ** 2).sum()
cost.name = 'cost_with_regularization'
from blocks.bricks import MLP
mlp = MLP(activations=[Rectifier(), Softmax()], dims=[784, 100, 10]).apply(x)
from blocks.initialization import IsotropicGaussian, Constant
input_to_hidden.weights_init = hidden_to_output.weights_init = IsotropicGaussian(0.01)
input_to_hidden.biases_init = hidden_to_output.biases_init = Constant(0)
input_to_hidden.initialize()
from cuboid.bricks import BatchNormalizationConv, BatchNormalization, Dropout
from cuboid.extensions import LogToFile
from blocks.filter import VariableFilter, get_brick
from blocks.model import Model
from blocks.graph import ComputationGraph
X = T.matrix("features")
o = X.reshape((X.shape[0], 1, 28, 28))
l = Convolutional(filter_size=(5, 5),
num_filters=32,
num_channels=1,
image_size=(28,28),
weights_init=IsotropicGaussian(std=0.01),
biases_init=IsotropicGaussian(std=0.01, mean=1.0),
use_bias=True,
border_mode="valid",
step=(1,1))
l.initialize()
o = l.apply(o)
l = BatchNormalizationConv(input_shape=l.get_dim("output"),
B_init=IsotropicGaussian(std=0.01),
Y_init=IsotropicGaussian(std=0.01))
l.initialize()
o = l.apply(o)
o = Rectifier().apply(o)
dropout_locs = []
# CONVOLUTION LAYERS
conv_in = inputt[:, None, :, None]
conv_in_channels = 1
conv_in_mask = input_mask
cb = []
for i, p in enumerate(convs):
# Convolution bricks
conv = Convolutional(
filter_size=(p["filter_size"], 1),
# step=(p['stride'],1),
num_filters=p["nfilter"],
num_channels=conv_in_channels,
batch_size=batch_size,
border_mode="valid",
tied_biases=True,
name="conv%d" % i,
)
cb.append(conv)
maxpool = MaxPooling(pooling_size=(p["pool_stride"], 1), name="mp%d" % i)
conv_out = conv.apply(conv_in)[:, :, :: p["stride"], :]
conv_out = maxpool.apply(conv_out)
if p["normalize"]:
conv_out_mean = conv_out.mean(axis=2).mean(axis=0)
conv_out_var = ((conv_out - conv_out_mean[None, :, None, :]) ** 2).mean(axis=2).mean(axis=0).sqrt()
conv_out = (conv_out - conv_out_mean[None, :, None, :]) / conv_out_var[None, :, None, :]
if p["activation"] is not None:
conv_out = p["activation"].apply(conv_out)
def __init__(self, *args, **kwargs):
self.mask_type = kwargs.pop('mask_type', None)
Convolutional.__init__(self, *args, **kwargs)
class ResidualConvolutional(Initializable):
@lazy(allocation=['filter_size', 'num_filters', 'num_channels'])
def __init__(self, filter_size, num_filters, num_channels,
batch_size=None,
mid_noise=False,
out_noise=False,
tied_noise=False,
tied_sigma=False,
noise_rate=None,
noise_batch_size=None,
prior_noise_level=None,
image_size=(None, None), step=(1, 1),
**kwargs):
self.filter_size = filter_size
self.num_filters = num_filters
self.batch_size = batch_size
self.num_channels = num_channels
self.image_size = image_size
self.mid_noise = mid_noise
self.noise_batch_size = noise_batch_size
self.noise_rate = noise_rate
self.step = step
self.border_mode = 'half'
self.tied_biases = True
depth = 2
self.b0 = SpatialBatchNormalization(name='b0')
self.r0 = Rectifier(name='r0')
self.n0 = (SpatialNoise(name='n0', noise_rate=self.noise_rate,
tied_noise=tied_noise, tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level) if mid_noise else None)
self.c0 = Convolutional(name='c0')
self.b1 = SpatialBatchNormalization(name='b1')
self.r1 = Rectifier(name='r1')
self.n1 = (SpatialNoise(name='n1', noise_rate=self.noise_rate,
tied_noise=tied_noise, tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level) if out_noise else None)
self.c1 = Convolutional(name='c1')
kwargs.setdefault('children', []).extend([c for c in [
self.c0, self.b0, self.r0, self.n0,
self.c1, self.b1, self.r1, self.n1] if c is not None])
super(ResidualConvolutional, self).__init__(**kwargs)
def get_dim(self, name):
if name == 'input_':
return ((self.num_channels,) + self.image_size)
if name == 'output':
return self.c1.get_dim(name)
return super(ResidualConvolutionalUnit, self).get_dim(name)
@property
def num_output_channels(self):
return self.num_filters
def _push_allocation_config(self):
self.b0.input_dim = self.get_dim('input_')
self.b0.push_allocation_config()
if self.r0:
self.r0.push_allocation_config()
if self.n0:
self.n0.noise_batch_size = self.noise_batch_size
self.n0.num_channels = self.num_channels
self.n0.image_size = self.image_size
self.c0.filter_size = self.filter_size
self.c0.batch_size = self.batch_size
self.c0.num_channels = self.num_channels
self.c0.num_filters = self.num_filters
self.c0.border_mode = self.border_mode
self.c0.image_size = self.image_size
self.c0.step = self.step
self.c0.use_bias = False
self.c0.push_allocation_config()
c0_shape = self.c0.get_dim('output')
self.b1.input_dim = c0_shape
self.b1.push_allocation_config()
self.r1.push_allocation_config()
if self.n1:
self.n1.noise_batch_size = self.noise_batch_size
self.n1.num_channels = self.num_filters
self.n1.image_size = c0_shape[1:]
self.c1.filter_size = self.filter_size
self.c1.batch_size = self.batch_size
self.c1.num_channels = self.num_filters
self.c1.num_filters = self.num_filters
self.c1.border_mode = self.border_mode
self.c1.image_size = c0_shape[1:]
self.c1.step = (1, 1)
self.c1.use_bias = False
self.c1.push_allocation_config()
@application(inputs=['input_'], outputs=['output'])
def apply(self, input_):
shortcut = input_
# Batchnorm, then Relu, then Convolution
first_conv = self.b0.apply(input_)
first_conv = self.r0.apply(first_conv)
if self.n0:
first_conv = self.n0.apply(first_conv)
first_conv = self.c0.apply(first_conv)
# Batchnorm, then Relu, then Convolution (second time)
second_conv = self.b1.apply(first_conv)
second_conv = self.r1.apply(second_conv)
if self.n1:
second_conv = self.n1.apply(second_conv)
residual = second_conv
# Apply stride and zero-padding to match shortcut to output
if self.step and self.step != (1, 1):
shortcut = shortcut[:,:,::self.step[0],::self.step[1]]
if self.num_filters > self.num_channels:
padshape = (residual.shape[0],
self.num_filters - self.num_channels,
residual.shape[2], residual.shape[3])
shortcut = tensor.concatenate(
[shortcut, tensor.zeros(padshape, dtype=residual.dtype)],
axis=1)
elif self.num_filters < self.num_channels:
shortcut = shortcut[:,:self.num_channels,:,:]
response = shortcut + residual
return response
