def residual_block(l, increase_dim=False, first=False, filters=16):
if increase_dim:
first_stride = (2, 2)
else:
first_stride = (1, 1)
if first:
# hacky solution to keep layers correct
bn_pre_relu = l
else:
# contains the BN -> ReLU portion, steps 1 to 2
bn_pre_conv = BatchNormLayer(l)
bn_pre_relu = NonlinearityLayer(bn_pre_conv, rectify)
# contains the weight -> BN -> ReLU portion, steps 3 to 5
conv_1 = batch_norm(
ConvLayer(bn_pre_relu,
num_filters=filters,
filter_size=(3, 3),
stride=first_stride,
nonlinearity=rectify,
pad='same',
W=HeNormal(gain='relu')))
dropout = DropoutLayer(conv_1, p=0.3)
# contains the last weight portion, step 6
conv_2 = ConvLayer(dropout,
num_filters=filters,
filter_size=(3, 3),
stride=(1, 1),
nonlinearity=None,
pad='same',
W=HeNormal(gain='relu'))
# add shortcut connections
if increase_dim:
# projection shortcut, as option B in paper
projection = ConvLayer(l,
num_filters=filters,
filter_size=(1, 1),
stride=(2, 2),
nonlinearity=None,
pad='same',
b=None)
block = ElemwiseSumLayer([conv_2, projection])
elif first:
# projection shortcut, as option B in paper
projection = ConvLayer(l,
num_filters=filters,
filter_size=(1, 1),
stride=(1, 1),
nonlinearity=None,
pad='same',
b=None)
block = ElemwiseSumLayer([conv_2, projection])
else:
block = ElemwiseSumLayer([conv_2, l])
return block
def contraction(depth, deepest):
n_filters = filter_for_depth(depth)
incoming = net['input'] if depth == 0 else net['pool{}'.format(depth -
1)]
net['conv{}_1'.format(depth)] = Conv2DLayer(incoming,
num_filters=n_filters,
filter_size=3,
pad='valid',
W=HeNormal(gain='relu'),
nonlinearity=nonlinearity)
net['conv{}_2'.format(depth)] = Conv2DLayer(
net['conv{}_1'.format(depth)],
num_filters=n_filters,
filter_size=3,
pad='valid',
W=HeNormal(gain='relu'),
nonlinearity=nonlinearity)
if P.BATCH_NORMALIZATION:
net['conv{}_2'.format(depth)] = batch_norm(
net['conv{}_2'.format(depth)],
alpha=P.BATCH_NORMALIZATION_ALPHA)
if not deepest:
net['pool{}'.format(depth)] = MaxPool2DLayer(
net['conv{}_2'.format(depth)], pool_size=2, stride=2)
def add_layer(incoming, num_channels, dropout):
layer = ScaleLayer(incoming)
layer = BiasLayer(layer)
# Bottleneck layer to reduce number of input channels to 4 times the number of output channels
layer = NonlinearityLayer(layer, nonlinearity=rectify)
layer = Conv2DLayer(layer,
num_filters=4 * num_channels,
filter_size=(1, 1),
stride=(1, 1),
W=HeNormal(gain='relu'), b=None,
flip_filters=False,
nonlinearity=None)
layer = BatchNormLayer(layer, beta=None, gamma=None)
if dropout > 0:
layer = DropoutLayer(layer, p=dropout)
# Convolutional layer (using padding to keep same dimensions)
layer = NonlinearityLayer(layer, nonlinearity=rectify)
layer = Conv2DLayer(layer,
num_filters=num_channels,
filter_size=(3, 3),
stride=(1, 1),
W=HeNormal(gain='relu'), b=None,
pad='same',
flip_filters=False,
nonlinearity=None)
layer = BatchNormLayer(layer, beta=None, gamma=None)
if dropout > 0:
layer = DropoutLayer(layer, p=dropout)
# Concatenate the input filters with the new filters
layer = ConcatLayer([incoming, layer], axis=1)
return layer
def test_he_normal_c01b_4d_only():
from lasagne.init import HeNormal
with pytest.raises(RuntimeError):
HeNormal(c01b=True).sample((100, ))
with pytest.raises(RuntimeError):
HeNormal(c01b=True).sample((100, 100))
with pytest.raises(RuntimeError):
HeNormal(c01b=True).sample((100, 100, 100))
def residual_block(l,
increase_dim=False,
projection=True,
first=False,
filters=16):
if increase_dim:
first_stride = (2, 2)
else:
first_stride = (1, 1)
if first:
bn_pre_relu = l
else:
bn_pre_conv = BatchNormLayer(l)
bn_pre_relu = NonlinearityLayer(bn_pre_conv, rectify)
conv_1 = batch_norm(
ConvLayer(bn_pre_relu,
num_filters=filters,
filter_size=(3, 3),
stride=first_stride,
nonlinearity=rectify,
pad='same',
W=HeNormal(gain='relu')))
dropout = DropoutLayer(conv_1, p=0.3)
conv_2 = ConvLayer(dropout,
num_filters=filters,
filter_size=(3, 3),
stride=(1, 1),
nonlinearity=None,
pad='same',
W=HeNormal(gain='relu'))
if increase_dim:
projection = ConvLayer(l,
num_filters=filters,
filter_size=(1, 1),
stride=(2, 2),
nonlinearity=None,
pad='same',
b=None)
block = ElemwiseSumLayer([conv_2, projection])
elif first:
projection = ConvLayer(l,
num_filters=filters,
filter_size=(1, 1),
stride=(1, 1),
nonlinearity=None,
pad='same',
b=None)
block = ElemwiseSumLayer([conv_2, projection])
else:
block = ElemwiseSumLayer([conv_2, l])
return block
def cifar_model(cls, n=9, incoming=None, classes=10, **kwargs):
model = incoming or InputLayer(shape=(None, 3, 32, 32))
builder = cls(model, **kwargs)
# first layer, output is 16 x 32 x 32
builder.model = builder.convolution(model, 16, init_gain=1.0)
# first stack of residual blocks, output is 16 x 32 x 32
for _ in range(n):
builder.add_residual_block(16)
# second stack of residual blocks, output is 32 x 16 x 16
builder.add_residual_block(32, dim_inc=True)
for _ in range(1, n):
builder.add_residual_block(32)
# third stack of residual blocks, output is 64 x 8 x 8
builder.add_residual_block(64, dim_inc=True)
for _ in range(1, n):
builder.add_residual_block(64)
model = builder.nonlinearity(BatchNormLayer(builder.model))
# average pooling
model = GlobalPoolLayer(model)
# fully connected layer
model = DenseLayer(model,
num_units=classes,
W=HeNormal(gain='relu'),
nonlinearity=softmax)
return model
def expansion(depth, deepest):
n_filters = filter_for_depth(depth)
incoming = net['conv{}_2'.format(depth + 1)] if deepest else net[
'_conv{}_2'.format(depth + 1)]
upscaling = Upscale2DLayer(incoming, 4)
net['upconv{}'.format(depth)] = Conv2DLayer(upscaling,
num_filters=n_filters,
filter_size=2,
stride=2,
W=HeNormal(gain='relu'),
nonlinearity=nonlinearity)
if P.SPATIAL_DROPOUT > 0:
bridge_from = DropoutLayer(net['conv{}_2'.format(depth)],
P.SPATIAL_DROPOUT)
else:
bridge_from = net['conv{}_2'.format(depth)]
net['bridge{}'.format(depth)] = ConcatLayer(
[net['upconv{}'.format(depth)], bridge_from],
axis=1,
cropping=[None, None, 'center', 'center'])
net['_conv{}_1'.format(depth)] = Conv2DLayer(
net['bridge{}'.format(depth)],
num_filters=n_filters,
filter_size=3,
pad='valid',
W=HeNormal(gain='relu'),
nonlinearity=nonlinearity)
#if P.BATCH_NORMALIZATION:
# net['_conv{}_1'.format(depth)] = batch_norm(net['_conv{}_1'.format(depth)])
if P.DROPOUT > 0:
net['_conv{}_1'.format(depth)] = DropoutLayer(
net['_conv{}_1'.format(depth)], P.DROPOUT)
net['_conv{}_2'.format(depth)] = Conv2DLayer(
net['_conv{}_1'.format(depth)],
num_filters=n_filters,
filter_size=3,
pad='valid',
W=HeNormal(gain='relu'),
nonlinearity=nonlinearity)
def test_he_normal_gain():
from lasagne.init import HeNormal
sample = HeNormal(gain=10.0).sample((100, 100))
assert -0.1 < sample.mean() < 0.1
assert 0.9 < sample.std() < 1.1
sample = HeNormal(gain='relu').sample((200, 50))
assert -0.1 < sample.mean() < 0.1
assert 0.07 < sample.std() < 0.12
def ResNet_FullPre_Wide(input_shape=(None, 3, PIXELS, PIXELS),
input_var=None,
n_classes=10,
n=6,
k=4):
"""
Adapted from https://github.com/Lasagne/Recipes/tree/master/papers/deep_residual_learning.
Tweaked to be consistent with 'Identity Mappings in Deep Residual Networks', Kaiming He et al. 2016 (https://arxiv.org/abs/1603.05027)
And 'Wide Residual Networks', Sergey Zagoruyko, Nikos Komodakis 2016 (http://arxiv.org/pdf/1605.07146v1.pdf)
Depth = 6n + 2
"""
n_filters = {0: 16, 1: 16 * k, 2: 32 * k, 3: 64 * k}
# Building the network
l_in = InputLayer(shape=input_shape, input_var=input_var)
# first layer, output is 16 x 64 x 64
l = batch_norm(
ConvLayer(l_in,
num_filters=n_filters[0],
filter_size=(3, 3),
stride=(1, 1),
nonlinearity=rectify,
pad='same',
W=he_norm))
# first stack of residual blocks, output is 32 x 64 x 64
l = residual_wide_block(l, first=True, filters=n_filters[1])
for _ in range(1, n):
l = residual_wide_block(l, filters=n_filters[1])
# second stack of residual blocks, output is 64 x 32 x 32
l = residual_wide_block(l, increase_dim=True, filters=n_filters[2])
for _ in range(1, (n + 2)):
l = residual_wide_block(l, filters=n_filters[2])
# third stack of residual blocks, output is 128 x 16 x 16
l = residual_wide_block(l, increase_dim=True, filters=n_filters[3])
for _ in range(1, (n + 2)):
l = residual_wide_block(l, filters=n_filters[3])
bn_post_conv = BatchNormLayer(l)
bn_post_relu = NonlinearityLayer(bn_post_conv, rectify)
# average pooling
avg_pool = GlobalPoolLayer(bn_post_relu)
# fully connected layer
network = DenseLayer(avg_pool,
num_units=n_classes,
W=HeNormal(),
nonlinearity=softmax)
return network
def lrelu_conv(l_in, feat_out, stride=1, filter_size=3):
l = NonlinearityLayer(l_in, nonlin)
return Conv3DLayer(l,
feat_out,
filter_size,
stride,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
def build_model(self, incoming):
net, params = super(ResNet_BottleNeck_FullPreActivation_Monoclass,
self).build_model(incoming)
net['probsout'] = lasagne.layers.DenseLayer(
net['featsout'],
W=HeNormal(),
num_units=self.out_size,
nonlinearity=lasagne.nonlinearities.softmax)
params = lasagne.layers.get_all_params(net['probsout'], trainable=True)
return net, params
def build_model(self, incoming):
net, params = super(ResNet_FullPre_Wide_Multiclass,
self).build_model(incoming)
net['probsout'] = lasagne.layers.DenseLayer(
net['featsout'],
W=HeNormal(),
num_units=self.out_size,
nonlinearity=lasagne.nonlinearities.sigmoid)
params = lasagne.layers.get_all_params(net['probsout'], trainable=True)
return net, params
def ResNet_BottleNeck_FullPreActivation(
input_shape=(None, 3, PIXELS, PIXELS), input_var=None, n_classes=10,
n=18):
'''
Adapted from https://github.com/Lasagne/Recipes/tree/master/papers/deep_residual_learning.
Tweaked to be consistent with 'Identity Mappings in Deep Residual Networks', Kaiming He et al. 2016 (https://arxiv.org/abs/1603.05027)
Judging from https://github.com/KaimingHe/resnet-1k-layers/blob/master/resnet-pre-act.lua.
Number of filters go 16 -> 64 -> 128 -> 256
Forumala to figure out depth: 9n + 2
'''
# Building the network
l_in = InputLayer(shape=input_shape, input_var=input_var)
# first layer, output is 16x16x16
l = batch_norm(
ConvLayer(l_in,
num_filters=16,
filter_size=(3, 3),
stride=(1, 1),
nonlinearity=rectify,
pad='same',
W=he_norm))
# first stack of residual blocks, output is 64x16x16
l = residual_bottleneck_block(l, first=True)
for _ in range(1, n):
l = residual_bottleneck_block(l)
# second stack of residual blocks, output is 128x8x8
l = residual_bottleneck_block(l, increase_dim=True)
for _ in range(1, n):
l = residual_bottleneck_block(l)
# third stack of residual blocks, output is 256x4x4
l = residual_bottleneck_block(l, increase_dim=True)
for _ in range(1, n):
l = residual_bottleneck_block(l)
bn_post_conv = BatchNormLayer(l)
bn_post_relu = NonlinearityLayer(bn_post_conv, rectify)
# average pooling
avg_pool = GlobalPoolLayer(bn_post_relu)
# fully connected layer
network = DenseLayer(avg_pool,
num_units=n_classes,
W=HeNormal(),
nonlinearity=softmax)
return network
def build_cnn(self):
# Input Layer
l_in = InputLayer(self.input_shape, input_var=self.input_var)
# Conv1
l_conv1 = Conv2DLayer(l_in, num_filters=NUM_FILTERS1, filter_size=3,
nonlinearity=rectify, W=HeNormal())
l_drop1 = spatial_dropout(l_conv1, DROPOUT)
# Conv2
l_conv2 = Conv2DLayer(l_drop1, num_filters=NUM_FILTERS2, filter_size=2,
stride=2, nonlinearity=rectify, W=HeNormal())
l_drop2 = spatial_dropout(l_conv2, 2*DROPOUT)
# Pool
l_max = MaxPool2DLayer(l_drop2, pool_size=(2, 2))
l_max = batch_norm(l_max)
# FC
l_dense = DenseLayer(l_max, num_units=NUM_FILTERS3, nonlinearity=rectify)
l_drop3 = dropout(l_dense, 4*DROPOUT)
# Softmax Output
l_out = DenseLayer(l_drop3, num_units=2, nonlinearity=softmax)
self.model = l_out
def conv_norm_lrelu(l_in, feat_out):
l = Conv3DLayer(l_in,
feat_out,
3,
1,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
if do_norm:
l = BatchNormLayer(l, axes=axes)
return NonlinearityLayer(l, nonlin)
def norm_lrelu_conv(l_in, feat_out, stride=1, filter_size=3):
if do_norm:
l_in = BatchNormLayer(l_in, axes=axes)
l = NonlinearityLayer(l_in, nonlin)
return Conv3DLayer(l,
feat_out,
filter_size,
stride,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
def make_net(W, H, size1=20, size2=15):
net = NeuralNet(
layers=[
('input', InputLayer),
('dense1', DenseLayer),
('dense2', DenseLayer),
('output', DenseLayer),
],
input_shape=(None, W * H),
dense1_num_units=size1,
dense1_nonlinearity=LeakyRectify(leakiness=0.1),
dense1_W=HeNormal(),
dense1_b=Constant(),
dense2_num_units=size2,
dense2_nonlinearity=LeakyRectify(leakiness=0.1),
dense2_W=HeNormal(),
dense2_b=Constant(),
output_num_units=4,
output_nonlinearity=softmax,
output_W=HeNormal(),
output_b=Constant(),
update=nesterov_momentum, # todo
update_learning_rate=shared(float32(1.)),
update_momentum=0.9,
max_epochs=200,
on_epoch_finished=[
StopWhenOverfitting(),
StopAfterMinimum(),
AdjustLearningRate(1., 0.0001),
],
#label_encoder = False,
regression=True,
verbose=1,
batch_iterator_train=BatchIterator(batch_size=128), # todo
batch_iterator_test=BatchIterator(batch_size=128),
train_split=TrainSplit(eval_size=0.1),
)
net.initialize()
return net
def getNet6():
inputLayer = layers.InputLayer(shape=(None, 1, imageShape[0], imageShape[1])) #120x120
conv1Layer = layers.Conv2DLayer(inputLayer, num_filters=32, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #120x120
conv2Layer = layers.Conv2DLayer(conv1Layer, num_filters=32, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #120x120
pool1Layer = layers.MaxPool2DLayer(conv2Layer, pool_size=(2,2)) #60x60
conv3Layer = layers.Conv2DLayer(pool1Layer, num_filters=64, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #60x60
conv4Layer = layers.Conv2DLayer(conv3Layer, num_filters=64, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #60x60
conv5Layer = layers.Conv2DLayer(conv4Layer, num_filters=64, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #60x60
pool2Layer = layers.MaxPool2DLayer(conv5Layer, pool_size=(2,2)) #30x30
conv6Layer = layers.Conv2DLayer(pool2Layer, num_filters=128, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #30x30
conv7Layer = layers.Conv2DLayer(conv6Layer, num_filters=128, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #30x30
conv8Layer = layers.Conv2DLayer(conv7Layer, num_filters=128, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #30x30
pool3Layer = layers.MaxPool2DLayer(conv8Layer, pool_size=(2,2)) #15x15
conv9Layer = layers.Conv2DLayer(pool3Layer, num_filters=256, filter_size=(4,4), W=HeNormal('relu'), nonlinearity=rectify) #12x12
flattenLayer = layers.FlattenLayer(conv9Layer)
hidden1Layer = layers.DenseLayer(flattenLayer, num_units=1024, W=HeNormal('relu'), nonlinearity=rectify)
dropout1Layer = layers.DropoutLayer(hidden1Layer, p=0.5)
hidden2Layer = layers.DenseLayer(dropout1Layer, num_units=512, W=HeNormal('relu'), nonlinearity=rectify)
dropout2Layer = layers.DropoutLayer(hidden2Layer, p=0.5)
hidden3Layer = layers.DenseLayer(dropout2Layer, num_units=256, W=HeNormal('relu'), nonlinearity=rectify)
dropout3Layer = layers.DropoutLayer(hidden3Layer, p=0.5)
hidden4Layer = layers.DenseLayer(dropout3Layer, num_units=128, W=HeNormal('relu'), nonlinearity=rectify)
outputLayer = layers.DenseLayer(hidden4Layer, num_units=10, W=HeNormal('relu'), nonlinearity=softmax)
return outputLayer
def define_network(inputs):
network = lasagne.layers.InputLayer(shape=(None, params.CHANNELS, params.INPUT_SIZE, params.INPUT_SIZE, params.INPUT_SIZE),
input_var=inputs)
network = Conv3DDNNLayer(
network, num_filters=64, filter_size=(5, 5, 5),
nonlinearity=lasagne.nonlinearities.leaky_rectify,
W=HeNormal(gain='relu'))
network = MaxPool3DDNNLayer(network, pool_size=(2, 2, 2))
if params.BATCH_NORMALIZATION:
network = lasagne.layers.batch_norm(network)
network = Conv3DDNNLayer(
network, num_filters=64, filter_size=(5, 5, 5),
nonlinearity=lasagne.nonlinearities.leaky_rectify,
W=HeNormal(gain='relu'))
network = Conv3DDNNLayer(
network, num_filters=96, filter_size=(5, 5, 5),
nonlinearity=lasagne.nonlinearities.leaky_rectify,
W=HeNormal(gain='relu'))
if params.BATCH_NORMALIZATION:
network = lasagne.layers.batch_norm(network)
network = lasagne.layers.DenseLayer(
network,
num_units=420,
nonlinearity=lasagne.nonlinearities.leaky_rectify,
W=HeNormal(gain='relu')
)
network = lasagne.layers.DenseLayer(
network, num_units=params.N_CLASSES,
nonlinearity=lasagne.nonlinearities.softmax)
return network
def get_network(self):
network = lasagne.layers.InputLayer(shape=(None, self.num_features),
input_var=self.input_var)
network = lasagne.layers.DropoutLayer(network, p=self.dropout)
network = DenseLayer(network,
nonlinearity=rectify,
num_units=self.num_nodes,
W=HeNormal(gain=.01))
for _ in xrange(0, self.num_layers):
network = self.add_residual_dense_maxout_block(
network, self.num_nodes, self.dropout)
return lasagne.layers.DenseLayer(
network, num_units=2, nonlinearity=lasagne.nonlinearities.softmax)
def build_model(self, img_batch, pose_code):
img_size = self.options['img_size']
pose_code_size = self.options['pose_code_size']
filter_size = self.options['filter_size']
batch_size = img_batch.shape[0]
# image encoding
l_in = InputLayer(shape = [None, img_size[0], img_size[1], img_size[2]], input_var=img_batch)
l_in_dimshuffle = DimshuffleLayer(l_in, (0,3,1,2))
l_conv1_1 = Conv2DLayer(l_in_dimshuffle, num_filters=64, filter_size=filter_size, W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2))
l_conv1_2 = Conv2DLayer(l_conv1_1, num_filters=64, filter_size=filter_size, W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2))
l_pool1 = MaxPool2DLayer(l_conv1_2, pool_size=(2,2))
# pose encoding
l_in_2 = InputLayer(shape=(None, pose_code_size), input_var=pose_code)
l_pose_1 = DenseLayer(l_in_2, num_units=512, W=HeNormal(),nonlinearity=rectify)
l_pose_2 = DenseLayer(l_pose_1, num_units=pose_code_size*l_pool1.output_shape[2]*l_pool1.output_shape[3], W=HeNormal(),nonlinearity=rectify)
l_pose_reshape = ReshapeLayer(l_pose_2, shape=(batch_size, pose_code_size, l_pool1.output_shape[2], l_pool1.output_shape[3]))
# deeper fusion
l_concat = ConcatLayer([l_pool1, l_pose_reshape], axis=1)
l_pose_conv_1 = Conv2DLayer(l_concat, num_filters=128, filter_size=filter_size, W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2))
l_pose_conv_2 = Conv2DLayer(l_pose_conv_1, num_filters=128, filter_size=filter_size, W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2))
l_pool2 = MaxPool2DLayer(l_pose_conv_2, pool_size=(2,2))
l_conv_3 = Conv2DLayer(l_pool2, num_filters=128, filter_size=(1,1), W=HeNormal())
l_unpool1 = Unpool2DLayer(l_conv_3, ds = (2,2))
# image decoding
l_deconv_conv1_1 = Conv2DLayer(l_unpool1, num_filters=128, filter_size=filter_size, nonlinearity=rectify,W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2))
l_deconv_conv1_2 = Conv2DLayer(l_deconv_conv1_1, num_filters=64, filter_size=filter_size, nonlinearity=rectify,W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2))
l_unpool2 = Unpool2DLayer(l_deconv_conv1_2, ds = (2,2))
l_deconv_conv2_1 = Conv2DLayer(l_unpool2, num_filters=64, filter_size=filter_size, nonlinearity=None, W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2))
l_deconv_conv2_2 = Conv2DLayer(l_deconv_conv2_1, num_filters=img_size[2], filter_size=filter_size, nonlinearity=None, W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2))
return l_deconv_conv2_2, l_pose_reshape
def cifar_model(cls, n=9, incoming=None, classes=10, **kwargs):
"""Create model for the CIFAR data set like in section 4.2.
Parameters
----------
n : integer (``9``)
A parameter used to govern the size of the network as
described in the paper.
incoming : a :class:`Layer` instance or ``None``
The input layer, if it is ``None`` a new one will be created.
classes : integer (`10``)
The number of classes to train, usually ``10`` or ``100``.
kwargs : key-word arguments
The key-word arguments that get passed down to the constructor.
Returns
-------
a :class:`DenseLayer` instance
The model in form of its last layer.
"""
model = incoming or InputLayer(shape=(None, 3, 32, 32))
builder = cls(model, **kwargs)
# first layer, output is 16 x 32 x 32
model = builder.convolution(model, 16, init_gain=1.0)
model = builder.nonlinearity(model)
builder.model = model
# first stack of residual blocks, output is 16 x 32 x 32
for _ in range(n):
builder.add_residual_block(16)
# second stack of residual blocks, output is 32 x 16 x 16
builder.add_residual_block(32, dim_inc=True)
for _ in range(1, n):
builder.add_residual_block(32)
# third stack of residual blocks, output is 64 x 8 x 8
builder.add_residual_block(64, dim_inc=True)
for _ in range(1, n):
builder.add_residual_block(64)
# average pooling
model = GlobalPoolLayer(builder.model)
# fully connected layer
model = DenseLayer(model,
num_units=classes,
W=HeNormal(gain='relu'),
nonlinearity=softmax)
return model
def ResNet_FullPreActivation(input_var=None, n=18):
"""
Adapted from https://github.com/Lasagne/Recipes/tree/master/papers/deep_residual_learning.
Tweaked to be consistent with 'Identity Mappings in Deep Residual Networks', Kaiming He et al. 2016 (https://arxiv.org/abs/1603.05027)
Formula to figure out depth: 6n + 2
"""
# Building the network
l_in = InputLayer(shape=(None, 3, PIXELS, PIXELS), input_var=input_var)
# first layer, output is 16 x 32 x 32
l = batch_norm(
ConvLayer(l_in,
num_filters=16,
filter_size=(3, 3),
stride=(1, 1),
nonlinearity=rectify,
pad='same',
W=he_norm))
# first stack of residual blocks, output is 16 x 32 x 32
l = residual_block(l, first=True)
for _ in range(1, n):
l = residual_block(l)
# second stack of residual blocks, output is 32 x 16 x 16
l = residual_block(l, increase_dim=True)
for _ in range(1, n):
l = residual_block(l)
# third stack of residual blocks, output is 64 x 8 x 8
l = residual_block(l, increase_dim=True)
for _ in range(1, n):
l = residual_block(l)
bn_post_conv = BatchNormLayer(l)
bn_post_relu = NonlinearityLayer(bn_post_conv, rectify)
# average pooling
avg_pool = GlobalPoolLayer(bn_post_relu)
# fully connected layer
network = DenseLayer(avg_pool,
num_units=10,
W=HeNormal(),
nonlinearity=softmax)
return network
def convolution(incoming,
num_filters,
filter_size=(3, 3),
stride=(1, 1),
init_gain='relu'):
"""Standard convolution method."""
return Conv2DLayer(incoming,
num_filters=num_filters,
filter_size=filter_size,
stride=stride,
nonlinearity=None,
pad='same',
W=HeNormal(gain=init_gain),
b=None,
flip_filters=False)
def _model_definition(self, net):
"""
Builds the architecture of the network
"""
he_norm = HeNormal(gain='relu')
# Input filtering and downsampling with max pooling
net = batch_norm(
Conv2DLayer(net,
num_filters=32,
filter_size=7,
pad='same',
nonlinearity=rectify,
W=he_norm))
net = MaxPool2DLayer(net, 2)
net = batch_norm(
Conv2DLayer(net,
num_filters=64,
filter_size=3,
pad='same',
nonlinearity=rectify,
W=he_norm))
net = MaxPool2DLayer(net, 2)
net = batch_norm(
Conv2DLayer(net,
num_filters=128,
filter_size=3,
pad='same',
nonlinearity=rectify,
W=he_norm))
net = batch_norm(
Conv2DLayer(net,
num_filters=128,
filter_size=3,
pad='same',
nonlinearity=rectify,
W=he_norm))
net = batch_norm(
DenseLayer(net, num_units=1024, nonlinearity=rectify, W=he_norm))
net = batch_norm(
DenseLayer(net, num_units=1024, nonlinearity=rectify, W=he_norm))
# Pooling
#net = MaxPool1DLayer(net, 1)
return net
def test_he_normal_gain():
from lasagne.init import HeNormal
sample = HeNormal(gain=10.0).sample((100, 100))
assert -0.1 < sample.mean() < 0.1
assert 0.9 < sample.std() < 1.1
sample = HeNormal(gain='relu').sample((200, 50))
assert -0.1 < sample.mean() < 0.1
assert 0.07 < sample.std() < 0.12
def norm_lrelu_upscale_conv_norm_lrelu(l_in, feat_out):
if do_norm:
l_in = BatchNormLayer(l_in, axes=axes)
l = NonlinearityLayer(l_in, nonlin)
l = Upscale3DLayer(l, 2)
l = Conv3DLayer(l,
feat_out,
3,
1,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
if do_norm:
l = BatchNormLayer(l, axes=axes)
l = NonlinearityLayer(l, nonlin)
return l
def fusion_predict(fusion_snapshot_path, out_test_data_path,
stage2_data_root_dir, submist_save_file_path, fixed_size):
# all adapters
adapters = []
adapters.append(adapter1((48, 48), 'fusion/fcn1/96.npz'))
adapters.append(adapter2((48, 48), 'fusion/fcn2/92.npz'))
adapters.append(adapter3((48, 48), 'fusion/fcn3/52.npz'))
adapters.append(adapter4((48, 48), 'fusion/fcn4/470.npz'))
adapters.append(adapter5((48, 48), 'fusion/fcn5/280.npz'))
adapters.append(adapter6((48, 48), 'fusion/fcn6/114.npz'))
# input tensor
pred = T.tensor4('pred')
location = T.vector('location')
resolution = T.matrix('resolution')
target_volume = T.fscalar('volume')
# fusion layers
l_in = InputLayer(shape=(None, len(adapters), fixed_size[0],
fixed_size[1]),
input_var=pred)
mid = Conv2DLayer(l_in, num_filters=1, filter_size=(1, 1), W=HeNormal())
l_out = GlobalPoolLayer(mid)
test_area = lasagne.layers.get_output(l_out, deterministic=True).flatten()
test_pred_volume = utee.build_volume2(test_area, location, resolution,
fixed_size)
test_fn = theano.function([pred, location, resolution],
[test_area, test_pred_volume])
area_fn = theano.function([pred], test_area)
if os.path.exists(fusion_snapshot_path):
with np.load(fusion_snapshot_path) as f:
param_values = [f['arr_{}'.format(i)] for i in range(len(f.files))]
print('resuming snapshot from {}'.format(fusion_snapshot_path))
param_cur = lasagne.layers.get_all_params(l_out)
assert len(param_cur) == len(param_values)
for p, v in zip(param_cur, param_values):
p.set_value(v)
else:
print("snapshot {} not found".format(fusion_snapshot_path))
bm.fusion_submit(stage2_data_root_dir, out_test_data_path, adapters,
area_fn, test_fn, fixed_size, submist_save_file_path)
def add_residual_dense_maxout_block(self,
network,
num_nodes=240,
dropout=0.5):
network = lasagne.layers.DropoutLayer(network, p=self.dropout)
identity = network
network = DenseLayer(network,
nonlinearity=rectify,
num_units=self.num_nodes,
W=HeNormal(gain=.01))
network = FeaturePoolLayer(incoming=network,
pool_size=2,
axis=1,
pool_function=theano.tensor.max)
return NonlinearityLayer(ElemwiseSumLayer(
[identity, network.input_layer]),
nonlinearity=rectify)
def image_net_model(cls, config, incoming=None, classes=1000, **kwargs):
# staring block
model = incoming or InputLayer(shape=(None, 3, 224, 224))
builder = cls(model, **kwargs)
model = builder.convolution(model,
64,
filter_size=(7, 7),
stride=(2, 2),
init_gain=1.0)
model = MaxPool2DLayer(model,
pool_size=(3, 3),
stride=(2, 2),
ignore_border=False)
builder.model = model
config = iter(config)
# no increasing dimensions on the first chuck
n, num_filters = next(config)
for _ in range(n):
builder.add_residual_block(num_filters)
# add other residual blocks
for n, num_filters in config:
builder.add_residual_block(num_filters, dim_inc=True)
for _ in range(1, n):
builder.add_residual_block(num_filters)
model = builder.nonlinearity(BatchNormLayer(builder.model))
# final part of the network
model = Pool2DLayer(model,
pool_size=(7, 7),
stride=(1, 1),
mode='average_exc_pad',
ignore_border=False)
model = DenseLayer(model,
num_units=classes,
W=HeNormal(gain='relu'),
nonlinearity=softmax)
return model
def add_transition(incoming, num_filters, dropout):
layer = ScaleLayer(incoming)
layer = BiasLayer(layer)
layer = NonlinearityLayer(layer, nonlinearity=rectify)
# Reduce the number of filters
layer = Conv2DLayer(layer,
num_filters=num_filters,
filter_size=(1, 1),
stride=(1, 1),
W=HeNormal(gain='relu'), b=None,
flip_filters=False,
nonlinearity=None)
if dropout > 0:
layer = DropoutLayer(layer, p=dropout)
# Pooling layer to reduce the last two dimensions by half
layer = Pool2DLayer(layer,
pool_size=(2, 2),
stride=(2, 2),
mode='average_exc_pad')
layer = BatchNormLayer(layer, beta=None, gamma=None)
return layer
def test_he_normal():
from lasagne.init import HeNormal
sample = HeNormal().sample((100, 100))
assert -0.01 < sample.mean() < 0.01
assert 0.09 < sample.std() < 0.11
def test_he_normal_receptive_field():
from lasagne.init import HeNormal
sample = HeNormal().sample((50, 50, 2))
assert -0.01 < sample.mean() < 0.01
assert 0.09 < sample.std() < 0.11
def test_he_normal_c01b():
from lasagne.init import HeNormal
sample = HeNormal(c01b=True).sample((25, 2, 2, 25))
assert -0.01 < sample.mean() < 0.01
assert 0.09 < sample.std() < 0.11
for p, udn in zip(tparams, updir_new)]
f_update = theano.function([], [], updates=updir_new + param_up,
on_unused_input='ignore',
name='rmsprop_f_update')
params = [zipped_grads, running_grads, running_grads2, updir]
return (f_grad_shared, f_update, params)
##################################################################
# Variables and Constants
MAX_ITERS = 150 # 450
CONV_EPS = 1e-5
DFLT_VDIM = 100
_HE_NORMAL = HeNormal()
HE_NORMAL = lambda x: floatX(_HE_NORMAL.sample(x))
_HE_UNIFORM = HeUniform()
HE_UNIFORM = lambda x: floatX(_HE_UNIFORM.sample(x))
_HE_UNIFORM_RELU = HeUniform(gain=np.sqrt(2))
HE_UNIFORM_RELU = lambda x: floatX(_HE_UNIFORM_RELU.sample(x))
_RELU_ALPHA = 0.
_HE_UNIFORM_LEAKY_RELU = HeUniform(
gain=np.sqrt(2. / (1 + (_RELU_ALPHA or 1e-6)**2)))
HE_UNIFORM_LEAKY_RELU = lambda x: \
floatX(_HE_UNIFORM_LEAKY_RELU.sample(x))
_ORTHOGONAL = Orthogonal()
