def check(*input_dim):
sbn = SpatialBatchNormalization(input_dim)
sbn.initialize()
x = theano.tensor.TensorType(theano.config.floatX,
[False] * (len(input_dim) + 1))()
y = sbn.apply(x)
rng = numpy.random.RandomState((2015, 12, 17))
input_ = random_unif(rng, (11,) + input_dim)
assert_equal(y.eval({x: input_}), input_)
def __init__(self,
filter_size,
num_filters,
num_channels,
batch_size=None,
mid_noise=False,
out_noise=False,
tied_noise=False,
tied_sigma=False,
noise_rate=None,
noise_batch_size=None,
prior_noise_level=None,
image_size=(None, None),
step=(1, 1),
**kwargs):
self.filter_size = filter_size
self.num_filters = num_filters
self.batch_size = batch_size
self.num_channels = num_channels
self.image_size = image_size
self.mid_noise = mid_noise
self.noise_batch_size = noise_batch_size
self.noise_rate = noise_rate
self.step = step
self.border_mode = 'half'
self.tied_biases = True
depth = 2
self.b0 = SpatialBatchNormalization(name='b0')
self.r0 = Rectifier(name='r0')
self.n0 = (SpatialNoise(name='n0',
noise_rate=self.noise_rate,
tied_noise=tied_noise,
tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level)
if mid_noise else None)
self.c0 = Convolutional(name='c0')
self.b1 = SpatialBatchNormalization(name='b1')
self.r1 = Rectifier(name='r1')
self.n1 = (SpatialNoise(name='n1',
noise_rate=self.noise_rate,
tied_noise=tied_noise,
tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level)
if out_noise else None)
self.c1 = Convolutional(name='c1')
kwargs.setdefault('children', []).extend([
c for c in [
self.c0, self.b0, self.r0, self.n0, self.c1, self.b1, self.r1,
self.n1
] if c is not None
])
super(ResidualConvolutional, self).__init__(**kwargs)
def test_raise_exception_spatial():
"""Test that SpatialBatchNormalization raises an expected exception."""
# Work around a stupid bug in nose2 that unpacks the tuple into
# separate arguments.
sbn1 = SpatialBatchNormalization((5,))
yield assert_raises, (ValueError, sbn1.allocate)
sbn2 = SpatialBatchNormalization(3)
yield assert_raises, (ValueError, sbn2.allocate)
def do_not_fail(*input_dim):
try:
sbn = SpatialBatchNormalization(input_dim)
sbn.allocate()
except ValueError:
assert False
# Work around a stupid bug in nose2 by passing as *args.
yield do_not_fail, 5, 4, 3
yield do_not_fail, 7, 6
yield do_not_fail, 3, 9, 2, 3
def __init__(self, filter_size, num_filters, num_channels,
batch_size=None,
mid_noise=False,
out_noise=False,
tied_noise=False,
tied_sigma=False,
noise_rate=None,
noise_batch_size=None,
prior_noise_level=None,
image_size=(None, None), step=(1, 1),
**kwargs):
self.filter_size = filter_size
self.num_filters = num_filters
self.batch_size = batch_size
self.num_channels = num_channels
self.image_size = image_size
self.mid_noise = mid_noise
self.noise_batch_size = noise_batch_size
self.noise_rate = noise_rate
self.step = step
self.border_mode = 'half'
self.tied_biases = True
depth = 2
self.b0 = SpatialBatchNormalization(name='b0')
self.r0 = Rectifier(name='r0')
self.n0 = (SpatialNoise(name='n0', noise_rate=self.noise_rate,
tied_noise=tied_noise, tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level) if mid_noise else None)
self.c0 = Convolutional(name='c0')
self.b1 = SpatialBatchNormalization(name='b1')
self.r1 = Rectifier(name='r1')
self.n1 = (SpatialNoise(name='n1', noise_rate=self.noise_rate,
tied_noise=tied_noise, tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level) if out_noise else None)
self.c1 = Convolutional(name='c1')
kwargs.setdefault('children', []).extend([c for c in [
self.c0, self.b0, self.r0, self.n0,
self.c1, self.b1, self.r1, self.n1] if c is not None])
super(ResidualConvolutional, self).__init__(**kwargs)
def __init__(self, conv_activations, num_channels, image_shape,
filter_sizes, feature_maps, conv_steps, pooling_sizes,
top_mlp_activations, top_mlp_dims, border_mode='valid', **kwargs):
self.num_channels = num_channels
self.image_shape = image_shape
self.top_mlp_activations = top_mlp_activations
self.top_mlp_dims = top_mlp_dims
self.border_mode = border_mode
conv_parameters = zip(filter_sizes, feature_maps, conv_steps)
# Construct convolutional, activation, and pooling layers with corresponding parameters
conv_layers = list(interleave([
(Convolutional(filter_size=filter_size,
num_filters=num_filter,
step=conv_step,
border_mode=self.border_mode,
name='conv_{}'.format(i))
for i, (filter_size, num_filter, conv_step)
in enumerate(conv_parameters)),
conv_activations,
(MaxPooling(size, name='pool_{}'.format(i))
for i, size in enumerate(pooling_sizes))]))
# Applying SpatialBatchNormalization to inputs
self.layers = [SpatialBatchNormalization()] + conv_layers
#self.layers = conv_layers
self.conv_sequence = ConvolutionalSequence(self.layers, num_channels,
image_size=image_shape)
# Construct a top MLP
self.top_mlp = MLP(top_mlp_activations, top_mlp_dims)
# We need to flatten the output of the last convolutional layer.
# This brick accepts a tensor of dimension (batch_size, ...) and
# returns a matrix (batch_size, features)
self.flattener = Flattener()
application_methods = [self.conv_sequence.apply, self.flattener.apply,
self.top_mlp.apply]
super(LeNet, self).__init__(application_methods, **kwargs)
y = tensor.lmatrix('targets')
################# Convolutional Sequence 1 #################
# conv_layers1 parameters
i = 0 #Sequence
j = 0 #Sub Layer
filter_size = [(7,7), (5,5), (2,2), (5,5)]
num_filter = [16, 32, 48, 64]
num_channels = 3
pooling_size = [(3,3), (2,2), (2,2)]
conv_step = (1,1)
border_mode = 'valid'
conv_layers1 = []
conv_layers1.append(SpatialBatchNormalization(name='spatialBN_{}'.format(i)))
conv_layers1.append(
Convolutional(
filter_size=filter_size[j],
num_filters=num_filter[j],
step=conv_step,
border_mode=border_mode,
name='conv_{}'.format(i)))
conv_layers1.append(BatchNormalization(name='BNconv_{}'.format(i)))
conv_layers1.append(conv_activation[0])
conv_layers1.append(MaxPooling(pooling_size[j], name='pool_{}'.format(i)))
i = i + 1 #Sequence
conv_layers1.append(
Convolutional(
filter_size=filter_size[j+1],
class ResidualConvolutional(Initializable):
@lazy(allocation=['filter_size', 'num_filters', 'num_channels'])
def __init__(self, filter_size, num_filters, num_channels,
batch_size=None,
mid_noise=False,
out_noise=False,
tied_noise=False,
tied_sigma=False,
noise_rate=None,
noise_batch_size=None,
prior_noise_level=None,
image_size=(None, None), step=(1, 1),
**kwargs):
self.filter_size = filter_size
self.num_filters = num_filters
self.batch_size = batch_size
self.num_channels = num_channels
self.image_size = image_size
self.mid_noise = mid_noise
self.noise_batch_size = noise_batch_size
self.noise_rate = noise_rate
self.step = step
self.border_mode = 'half'
self.tied_biases = True
depth = 2
self.b0 = SpatialBatchNormalization(name='b0')
self.r0 = Rectifier(name='r0')
self.n0 = (SpatialNoise(name='n0', noise_rate=self.noise_rate,
tied_noise=tied_noise, tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level) if mid_noise else None)
self.c0 = Convolutional(name='c0')
self.b1 = SpatialBatchNormalization(name='b1')
self.r1 = Rectifier(name='r1')
self.n1 = (SpatialNoise(name='n1', noise_rate=self.noise_rate,
tied_noise=tied_noise, tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level) if out_noise else None)
self.c1 = Convolutional(name='c1')
kwargs.setdefault('children', []).extend([c for c in [
self.c0, self.b0, self.r0, self.n0,
self.c1, self.b1, self.r1, self.n1] if c is not None])
super(ResidualConvolutional, self).__init__(**kwargs)
def get_dim(self, name):
if name == 'input_':
return ((self.num_channels,) + self.image_size)
if name == 'output':
return self.c1.get_dim(name)
return super(ResidualConvolutionalUnit, self).get_dim(name)
@property
def num_output_channels(self):
return self.num_filters
def _push_allocation_config(self):
self.b0.input_dim = self.get_dim('input_')
self.b0.push_allocation_config()
if self.r0:
self.r0.push_allocation_config()
if self.n0:
self.n0.noise_batch_size = self.noise_batch_size
self.n0.num_channels = self.num_channels
self.n0.image_size = self.image_size
self.c0.filter_size = self.filter_size
self.c0.batch_size = self.batch_size
self.c0.num_channels = self.num_channels
self.c0.num_filters = self.num_filters
self.c0.border_mode = self.border_mode
self.c0.image_size = self.image_size
self.c0.step = self.step
self.c0.use_bias = False
self.c0.push_allocation_config()
c0_shape = self.c0.get_dim('output')
self.b1.input_dim = c0_shape
self.b1.push_allocation_config()
self.r1.push_allocation_config()
if self.n1:
self.n1.noise_batch_size = self.noise_batch_size
self.n1.num_channels = self.num_filters
self.n1.image_size = c0_shape[1:]
self.c1.filter_size = self.filter_size
self.c1.batch_size = self.batch_size
self.c1.num_channels = self.num_filters
self.c1.num_filters = self.num_filters
self.c1.border_mode = self.border_mode
self.c1.image_size = c0_shape[1:]
self.c1.step = (1, 1)
self.c1.use_bias = False
self.c1.push_allocation_config()
@application(inputs=['input_'], outputs=['output'])
def apply(self, input_):
shortcut = input_
# Batchnorm, then Relu, then Convolution
first_conv = self.b0.apply(input_)
first_conv = self.r0.apply(first_conv)
if self.n0:
first_conv = self.n0.apply(first_conv)
first_conv = self.c0.apply(first_conv)
# Batchnorm, then Relu, then Convolution (second time)
second_conv = self.b1.apply(first_conv)
second_conv = self.r1.apply(second_conv)
if self.n1:
second_conv = self.n1.apply(second_conv)
residual = second_conv
# Apply stride and zero-padding to match shortcut to output
if self.step and self.step != (1, 1):
shortcut = shortcut[:,:,::self.step[0],::self.step[1]]
if self.num_filters > self.num_channels:
padshape = (residual.shape[0],
self.num_filters - self.num_channels,
residual.shape[2], residual.shape[3])
shortcut = tensor.concatenate(
[shortcut, tensor.zeros(padshape, dtype=residual.dtype)],
axis=1)
elif self.num_filters < self.num_channels:
shortcut = shortcut[:,:self.num_channels,:,:]
response = shortcut + residual
return response
def do_not_fail(*input_dim):
try:
sbn = SpatialBatchNormalization(input_dim)
sbn.allocate()
except ValueError:
assert False
class ResidualConvolutional(Initializable):
@lazy(allocation=['filter_size', 'num_filters', 'num_channels'])
def __init__(self,
filter_size,
num_filters,
num_channels,
batch_size=None,
mid_noise=False,
out_noise=False,
tied_noise=False,
tied_sigma=False,
noise_rate=None,
noise_batch_size=None,
prior_noise_level=None,
image_size=(None, None),
step=(1, 1),
**kwargs):
self.filter_size = filter_size
self.num_filters = num_filters
self.batch_size = batch_size
self.num_channels = num_channels
self.image_size = image_size
self.mid_noise = mid_noise
self.noise_batch_size = noise_batch_size
self.noise_rate = noise_rate
self.step = step
self.border_mode = 'half'
self.tied_biases = True
depth = 2
self.b0 = SpatialBatchNormalization(name='b0')
self.r0 = Rectifier(name='r0')
self.n0 = (SpatialNoise(name='n0',
noise_rate=self.noise_rate,
tied_noise=tied_noise,
tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level)
if mid_noise else None)
self.c0 = Convolutional(name='c0')
self.b1 = SpatialBatchNormalization(name='b1')
self.r1 = Rectifier(name='r1')
self.n1 = (SpatialNoise(name='n1',
noise_rate=self.noise_rate,
tied_noise=tied_noise,
tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level)
if out_noise else None)
self.c1 = Convolutional(name='c1')
kwargs.setdefault('children', []).extend([
c for c in [
self.c0, self.b0, self.r0, self.n0, self.c1, self.b1, self.r1,
self.n1
] if c is not None
])
super(ResidualConvolutional, self).__init__(**kwargs)
def get_dim(self, name):
if name == 'input_':
return ((self.num_channels, ) + self.image_size)
if name == 'output':
return self.c1.get_dim(name)
return super(ResidualConvolutionalUnit, self).get_dim(name)
@property
def num_output_channels(self):
return self.num_filters
def _push_allocation_config(self):
self.b0.input_dim = self.get_dim('input_')
self.b0.push_allocation_config()
if self.r0:
self.r0.push_allocation_config()
if self.n0:
self.n0.noise_batch_size = self.noise_batch_size
self.n0.num_channels = self.num_channels
self.n0.image_size = self.image_size
self.c0.filter_size = self.filter_size
self.c0.batch_size = self.batch_size
self.c0.num_channels = self.num_channels
self.c0.num_filters = self.num_filters
self.c0.border_mode = self.border_mode
self.c0.image_size = self.image_size
self.c0.step = self.step
self.c0.use_bias = False
self.c0.push_allocation_config()
c0_shape = self.c0.get_dim('output')
self.b1.input_dim = c0_shape
self.b1.push_allocation_config()
self.r1.push_allocation_config()
if self.n1:
self.n1.noise_batch_size = self.noise_batch_size
self.n1.num_channels = self.num_filters
self.n1.image_size = c0_shape[1:]
self.c1.filter_size = self.filter_size
self.c1.batch_size = self.batch_size
self.c1.num_channels = self.num_filters
self.c1.num_filters = self.num_filters
self.c1.border_mode = self.border_mode
self.c1.image_size = c0_shape[1:]
self.c1.step = (1, 1)
self.c1.use_bias = False
self.c1.push_allocation_config()
@application(inputs=['input_'], outputs=['output'])
def apply(self, input_):
shortcut = input_
# Batchnorm, then Relu, then Convolution
first_conv = self.b0.apply(input_)
first_conv = self.r0.apply(first_conv)
if self.n0:
first_conv = self.n0.apply(first_conv)
first_conv = self.c0.apply(first_conv)
# Batchnorm, then Relu, then Convolution (second time)
second_conv = self.b1.apply(first_conv)
second_conv = self.r1.apply(second_conv)
if self.n1:
second_conv = self.n1.apply(second_conv)
residual = second_conv
# Apply stride and zero-padding to match shortcut to output
if self.step and self.step != (1, 1):
shortcut = shortcut[:, :, ::self.step[0], ::self.step[1]]
if self.num_filters > self.num_channels:
padshape = (residual.shape[0],
self.num_filters - self.num_channels,
residual.shape[2], residual.shape[3])
shortcut = tensor.concatenate(
[shortcut,
tensor.zeros(padshape, dtype=residual.dtype)],
axis=1)
elif self.num_filters < self.num_channels:
shortcut = shortcut[:, :self.num_channels, :, :]
response = shortcut + residual
return response
def __init__(self,
image_size=None,
output_size=None,
mid_noise=False,
out_noise=False,
tied_noise=False,
tied_sigma=False,
final_noise=False,
noise_batch_size=None,
noise_rate=None,
prior_noise_level=None,
**kwargs):
self.num_channels = 3
self.image_size = image_size or (32, 32)
self.output_size = output_size or 10
self.noise_batch_size = noise_batch_size
self.noise_rate = noise_rate
n = 18
num_filters = [16, 32, 64]
num_channels = num_filters[0]
self.convolutions = [
Convolutional(filter_size=(3, 3),
num_filters=num_channels,
step=(1, 1),
border_mode='half',
tied_biases=True,
name='conv_0'),
]
for j, num in enumerate(num_filters):
for i in range(n):
self.convolutions.append(
ResidualConvolutional(filter_size=(3, 3),
num_filters=num,
num_channels=num_channels,
mid_noise=mid_noise,
out_noise=out_noise,
tied_noise=tied_noise,
tied_sigma=tied_sigma,
noise_rate=noise_rate,
noise_batch_size=noise_batch_size,
prior_noise_level=prior_noise_level,
step=(2, 2) if i == 0 and j > 0 else
(1, 1),
name='group_%d_%d' % (num, i)))
num_channels = num
self.convolutions.extend([
SpatialBatchNormalization(name='bn_last'),
Rectifier(name='relu_last')
])
if final_noise:
self.convolutions.append(
SpatialNoise(name='nfin',
noise_rate=noise_rate,
noise_batch_size=noise_batch_size,
tied_noise=tied_noise,
tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level))
self.conv_sequence = ConvolutionalSequence(
self.convolutions,
image_size=self.image_size,
num_channels=self.num_channels)
# The AllConvNet applies average pooling to combine top-level
# features across the image.
self.flattener = GlobalAverageFlattener()
# Then it inserts one final 10-way FC layer before softmax
self.top_mlp = MLP([Softmax()], [num_filters[-1], self.output_size])
# self.top_softmax = Softmax()
application_methods = [
self.conv_sequence.apply, self.flattener.apply, self.top_mlp.apply
]
super(ResNet, self).__init__(application_methods, **kwargs)
