def add_layer(incoming, num_channels, dropout):
layer = ScaleLayer(incoming)
layer = BiasLayer(layer)
# Bottleneck layer to reduce number of input channels to 4 times the number of output channels
layer = NonlinearityLayer(layer, nonlinearity=rectify)
layer = Conv2DLayer(layer,
num_filters=4 * num_channels,
filter_size=(1, 1),
stride=(1, 1),
W=HeNormal(gain='relu'),
b=None,
flip_filters=False,
nonlinearity=None)
layer = BatchNormLayer(layer, beta=None, gamma=None)
if dropout > 0:
layer = DropoutLayer(layer, p=dropout)
# Convolutional layer (using padding to keep same dimensions)
layer = NonlinearityLayer(layer, nonlinearity=rectify)
layer = Conv2DLayer(layer,
num_filters=num_channels,
filter_size=(3, 3),
stride=(1, 1),
W=HeNormal(gain='relu'),
b=None,
pad='same',
flip_filters=False,
nonlinearity=None)
layer = BatchNormLayer(layer, beta=None, gamma=None)
if dropout > 0:
layer = DropoutLayer(layer, p=dropout)
# Concatenate the input filters with the new filters
layer = ConcatLayer([incoming, layer], axis=1)
return layer
def G_paper(
num_channels=1, # Overridden based on dataset.
resolution=32, # Overridden based on dataset.
label_size=0, # Overridden based on dataset.
fmap_base=4096,
fmap_decay=1.0,
fmap_max=256,
latent_size=None,
normalize_latents=True,
use_wscale=True,
use_pixelnorm=True,
use_leakyrelu=True,
use_batchnorm=False,
tanh_at_end=None,
**kwargs):
R = int(np.log2(resolution))
assert resolution == 2**R and resolution >= 4
cur_lod = theano.shared(np.float32(0.0))
def nf(stage):
return min(int(fmap_base / (2.0**(stage * fmap_decay))), fmap_max)
def PN(layer):
return PixelNormLayer(layer, name=layer.name +
'pn') if use_pixelnorm else layer
def BN(layer):
return lasagne.layers.batch_norm(layer) if use_batchnorm else layer
def WS(layer):
return WScaleLayer(layer, name=layer.name +
'S') if use_wscale else layer
if latent_size is None: latent_size = nf(0)
(act, iact) = (lrelu, ilrelu) if use_leakyrelu else (relu, irelu)
input_layers = [InputLayer(name='Glatents', shape=[None, latent_size])]
net = input_layers[-1]
if normalize_latents:
net = PixelNormLayer(net, name='Glnorm')
if label_size:
input_layers += [InputLayer(name='Glabels', shape=[None, label_size])]
net = ConcatLayer(name='Gina', incomings=[net, input_layers[-1]])
net = ReshapeLayer(name='Ginb', incoming=net, shape=[[0], [1], 1, 1])
net = PN(
BN(
WS(
Conv2DLayer(net,
name='G1a',
num_filters=nf(1),
filter_size=4,
pad='full',
nonlinearity=act,
W=iact))))
net = PN(
BN(
WS(
Conv2DLayer(net,
name='G1b',
num_filters=nf(1),
filter_size=3,
pad=1,
nonlinearity=act,
W=iact))))
lods = [net]
for I in xrange(2, R): # I = 2, 3, ..., R-1
net = Upscale2DLayer(net, name='G%dup' % I, scale_factor=2)
net = PN(
BN(
WS(
Conv2DLayer(net,
name='G%da' % I,
num_filters=nf(I),
filter_size=3,
pad=1,
nonlinearity=act,
W=iact))))
net = PN(
BN(
WS(
Conv2DLayer(net,
name='G%db' % I,
num_filters=nf(I),
filter_size=3,
pad=1,
nonlinearity=act,
W=iact))))
lods += [net]
lods = [
WS(
NINLayer(l,
name='Glod%d' % i,
num_units=num_channels,
nonlinearity=linear,
W=ilinear)) for i, l in enumerate(reversed(lods))
]
output_layer = LODSelectLayer(name='Glod',
incomings=lods,
cur_lod=cur_lod,
first_incoming_lod=0)
if tanh_at_end is not None:
output_layer = NonlinearityLayer(output_layer,
name='Gtanh',
nonlinearity=tanh)
if tanh_at_end != 1.0:
output_layer = non_trainable(
ScaleLayer(output_layer,
name='Gtanhs',
scales=lasagne.init.Constant(tanh_at_end)))
return dict(input_layers=input_layers,
output_layers=[output_layer],
cur_lod=cur_lod)
def build_densenet(
input_var,
input_shape=(None, 3, 224, 224),
num_filters_init=64,
growth_rate=32,
dropout=0.2,
num_classes=1000,
stages=[6, 12, 24, 16]):
if input_shape[2] % (2 ** len(stages)) != 0:
raise ValueError("input_shape[2] must be a multiple of {}.".format(2 ** len(stages)))
if input_shape[3] % (2 ** len(stages)) != 0:
raise ValueError("input_shape[3] must be a multiple of {}.".format(2 ** len(stages)))
# Input should be (BATCH_SIZE, NUM_CHANNELS, WIDTH, HEIGHT)
# NUM_CHANNELS is usually 3 (R,G,B) and for the ImageNet example the width and height are 224
network = InputLayer(input_shape, input_var)
# Apply 2D convolutions with a 7x7 filter (pad by 3 on each side)
# Because of the 2x2 stride the shape of the last two dimensions will be half the size of the input (112x112)
network = Conv2DLayer(network,
num_filters=num_filters_init,
filter_size=(7, 7),
stride=(2, 2),
pad=(3, 3),
W=HeNormal(gain='relu'), b=None,
flip_filters=False,
nonlinearity=None)
# Batch normalize
network = BatchNormLayer(network, beta=None, gamma=None)
# If dropout is enabled, apply after every convolutional and dense layer
if dropout > 0:
network = DropoutLayer(network, p=dropout)
# Apply ReLU
network = NonlinearityLayer(network, nonlinearity=rectify)
# Keep the maximum value of a 3x3 pool with a 2x2 stride
# This operation again divides the size of the last two dimensions by two (56x56)
network = MaxPool2DLayer(network,
pool_size=(3, 3),
stride=(2, 2),
pad=(1, 1))
# Add dense blocks
for i, num_layers in enumerate(stages):
# Except for the first block, we add a transition layer before the dense block that halves the number of filters, width and height
if i > 0:
network = add_transition(network, math.floor(network.output_shape[1] / 2), dropout)
network = build_block(network, num_layers, growth_rate, dropout)
# Apply global pooling and add a fully connected layer with softmax function
network = ScaleLayer(network)
network = BiasLayer(network)
network = NonlinearityLayer(network, nonlinearity=rectify)
network = GlobalPoolLayer(network)
network = DenseLayer(network,
num_units=num_classes,
W=HeNormal(gain=1),
nonlinearity=softmax)
return network
def build_densenet(input_shape=(None, 3, 32, 32),
input_var=None,
classes=10,
depth=40,
first_output=16,
growth_rate=12,
num_blocks=3,
dropout=0):
"""
Creates a DenseNet model in Lasagne.
Parameters
----------
input_shape : tuple
The shape of the input layer, as ``(batchsize, channels, rows, cols)``.
Any entry except ``channels`` can be ``None`` to indicate free size.
input_var : Theano expression or None
Symbolic input variable. Will be created automatically if not given.
classes : int
The number of classes of the softmax output.
depth : int
Depth of the network. Must be ``num_blocks * n + 1`` for some ``n``.
(Parameterizing by depth rather than n makes it easier to follow the
paper.)
first_output : int
Number of channels of initial convolution before entering the first
dense block, should be of comparable size to `growth_rate`.
growth_rate : int
Number of feature maps added per layer.
num_blocks : int
Number of dense blocks (defaults to 3, as in the original paper).
dropout : float
The dropout rate. Set to zero (the default) to disable dropout.
batchsize : int or None
The batch size to build the model for, or ``None`` (the default) to
allow any batch size.
inputsize : int, tuple of int or None
Returns
-------
network : Layer instance
Lasagne Layer instance for the output layer.
References
----------
.. [1] Gao Huang et al. (2016):
Densely Connected Convolutional Networks.
https://arxiv.org/abs/1608.06993
"""
if (depth - 1) % num_blocks != 0:
raise ValueError("depth must be num_blocks * n + 1 for some n")
# input and initial convolution
network = InputLayer(input_shape, input_var, name='input')
network = Conv2DLayer(network,
first_output,
3,
pad='same',
W=lasagne.init.HeNormal(gain='relu'),
b=None,
nonlinearity=None,
name='pre_conv')
network = BatchNormLayer(network, name='pre_bn', beta=None, gamma=None)
# note: The authors' implementation does *not* have a dropout after the
# initial convolution. This was missing in the paper, but important.
# if dropout:
# network = DropoutLayer(network, dropout)
# dense blocks with transitions in between
n = (depth - 1) // num_blocks
for b in range(num_blocks):
network = dense_block(network,
n - 1,
growth_rate,
dropout,
name_prefix='block%d' % (b + 1))
if b < num_blocks - 1:
network = transition(network,
dropout,
name_prefix='block%d_trs' % (b + 1))
# post processing until prediction
network = ScaleLayer(network, name='post_scale')
network = BiasLayer(network, name='post_shift')
network = NonlinearityLayer(network,
nonlinearity=rectify,
name='post_relu')
network = GlobalPoolLayer(network, name='post_pool')
network = DenseLayer(network,
classes,
nonlinearity=softmax,
W=lasagne.init.HeNormal(gain=1),
name='output')
return network
def batchnorm_pt2(incoming):
"""2nd part of batch normalization: scaling + biases."""
return BiasLayer(ScaleLayer(incoming))
def build_decoder(net):
net['uconv5_3'] = ConvLayer(net['conv5_3'], 512, 3, pad=1)
print "uconv5_3: {}".format(net['uconv5_3'].output_shape[1:])
net['uconv5_2'] = ConvLayer(net['uconv5_3'], 512, 3, pad=1)
print "uconv5_2: {}".format(net['uconv5_2'].output_shape[1:])
net['uconv5_1'] = ConvLayer(net['uconv5_2'], 512, 3, pad=1)
print "uconv5_1: {}".format(net['uconv5_1'].output_shape[1:])
net['upool4'] = Upscale2DLayer(net['uconv5_1'], scale_factor=2)
print "upool4: {}".format(net['upool4'].output_shape[1:])
net['uconv4_3'] = ConvLayer(net['upool4'], 512, 3, pad=1)
print "uconv4_3: {}".format(net['uconv4_3'].output_shape[1:])
net['uconv4_2'] = ConvLayer(net['uconv4_3'], 512, 3, pad=1)
print "uconv4_2: {}".format(net['uconv4_2'].output_shape[1:])
net['uconv4_1'] = ConvLayer(net['uconv4_2'], 512, 3, pad=1)
print "uconv4_1: {}".format(net['uconv4_1'].output_shape[1:])
net['upool3'] = Upscale2DLayer(net['uconv4_1'], scale_factor=2)
print "upool3: {}".format(net['upool3'].output_shape[1:])
net['uconv3_3'] = ConvLayer(net['upool3'], 256, 3, pad=1)
print "uconv3_3: {}".format(net['uconv3_3'].output_shape[1:])
net['uconv3_2'] = ConvLayer(net['uconv3_3'], 256, 3, pad=1)
print "uconv3_2: {}".format(net['uconv3_2'].output_shape[1:])
net['uconv3_1'] = ConvLayer(net['uconv3_2'], 256, 3, pad=1)
print "uconv3_1: {}".format(net['uconv3_1'].output_shape[1:])
net['upool2'] = Upscale2DLayer(net['uconv3_1'], scale_factor=2)
print "upool2: {}".format(net['upool2'].output_shape[1:])
net['uconv2_2'] = ConvLayer(net['upool2'], 128, 3, pad=1)
print "uconv2_2: {}".format(net['uconv2_2'].output_shape[1:])
net['uconv2_1'] = ConvLayer(net['uconv2_2'], 128, 3, pad=1)
print "uconv2_1: {}".format(net['uconv2_1'].output_shape[1:])
net['upool1'] = Upscale2DLayer(net['uconv2_1'], scale_factor=2)
print "upool1: {}".format(net['upool1'].output_shape[1:])
net['uconv1_2'] = ConvLayer(
net['upool1'],
64,
3,
pad=1,
)
print "uconv1_2: {}".format(net['uconv1_2'].output_shape[1:])
net['uconv1_1'] = ConvLayer(net['uconv1_2'], 64, 3, pad=1)
print "uconv1_1: {}".format(net['uconv1_1'].output_shape[1:])
net['output_encoder'] = ConvLayer(net['uconv1_1'],
3,
1,
pad=0,
nonlinearity=tanh)
print "output_encoder: {}".format(net['output_encoder'].output_shape[1:])
net['output_encoder_bias'] = BiasLayer(net['output_encoder'],
b=lasagne.init.Constant(1))
print "output_encoder_bias: {}".format(
net['output_encoder_bias'].output_shape[1:])
net['output_encoder_scaled'] = ScaleLayer(
net['output_encoder_bias'], scales=lasagne.init.Constant(127.5))
print "output_encoder_scaled: {}".format(
net['output_encoder_scaled'].output_shape[1:])
return net
def build_densenet(l_in,
input_var=None,
first_output=64,
growth_rate=32,
num_blocks=4,
dropout=0):
"""
Creates a DenseNet model in Lasagne.
Parameters
----------
input_shape : tuple
The shape of the input layer, as ``(batchsize, channels, rows, cols)``.
Any entry except ``channels`` can be ``None`` to indicate free size.
input_var : Theano expression or None
Symbolic input variable. Will be created automatically if not given.
classes : int
The number of classes of the softmax output.
first_output : int
Number of channels of initial convolution before entering the first
dense block, should be of comparable size to `growth_rate`.
growth_rate : int
Number of feature maps added per layer.
num_blocks : int
Number of dense blocks (defaults to 3, as in the original paper).
dropout : float
The dropout rate. Set to zero (the default) to disable dropout.
batchsize : int or None
The batch size to build the model for, or ``None`` (the default) to
allow any batch size.
inputsize : int, tuple of int or None
Returns
-------
network : Layer instance
Lasagne Layer instance for the output layer.
References
----------
.. [1] Gao Huang et al. (2016):
Densely Connected Convolutional Networks.
https://arxiv.org/abs/1608.06993
"""
nb_layers = [6, 12, 32, 32] # For DenseNet-169
nb_layers = [6, 12, 24, 16] # For DenseNet-121
# initial convolution
network = Conv2DLayer(l_in,
first_output,
filter_size=7,
stride=2,
pad='same',
W=lasagne.init.HeNormal(gain='relu'),
b=None,
nonlinearity=None,
name='pre_conv')
network = BatchNormLayer(network, name='pre_bn', beta=None, gamma=None)
network = ScaleLayer(network, name='pre_scale')
network = BiasLayer(network, name='pre_shift')
network = dnn.MaxPool2DDNNLayer(network, pool_size=3, stride=2)
# note: The authors' implementation does *not* have a dropout after the
# initial convolution. This was missing in the paper, but important.
# if dropout:
# network = DropoutLayer(network, dropout)
# dense blocks with transitions in between
for b in range(num_blocks):
network = dense_block(network,
nb_layers[b],
growth_rate,
dropout,
name_prefix='block%d' % (b + 1))
if b < num_blocks - 1:
network = transition(network,
dropout,
name_prefix='block%d_trs' % (b + 1))
# post processing until prediction
network = ScaleLayer(network, name='post_scale')
network = BiasLayer(network, name='post_shift')
network = NonlinearityLayer(network,
nonlinearity=rectify,
name='post_relu')
return network