def MDCL(incoming, num_filters, scales, name):
# Total number of layers
# W = theano.shared(lasagne.utils.floatX(Orthogonal(
winit = initmethod(0.02)
sinit = lasagne.init.Constant(1.0 / (1 + len(scales)))
# Number of incoming channels
ni = lasagne.layers.get_output_shape(incoming)[1]
# get weight parameter for this layer
W = theano.shared(lasagne.utils.floatX(
winit.sample((num_filters,
lasagne.layers.get_output_shape(incoming)[1], 3, 3))),
name=name + 'W')
n = C2D(incoming=incoming,
num_filters=num_filters,
filter_size=[3, 3],
stride=[1, 1],
pad=(1, 1),
W=W *
theano.shared(lasagne.utils.floatX(sinit.sample(num_filters)),
name + '_coeff_base').dimshuffle(0, 'x', 'x', 'x'),
b=None,
nonlinearity=None,
name=name + 'base')
# nc = [theano.shared(lasagne.utils.floatX(1.0/(1+len(scales))), name+'coeff_base')]
nd = []
for i, scale in enumerate(scales):
if scale == 0:
nd.append(
C2D(incoming=incoming,
num_filters=num_filters,
filter_size=[1, 1],
stride=[1, 1],
pad=(0, 0),
W=T.mean(W, axis=[2, 3]).dimshuffle(0, 1, 'x', 'x') *
theano.shared(
lasagne.utils.floatX(sinit.sample(num_filters)),
name + '_coeff_1x1').dimshuffle(0, 'x', 'x', 'x'),
b=None,
nonlinearity=None,
name=name + str(scale)))
else:
nd.append(
lasagne.layers.DilatedConv2DLayer(
incoming=lasagne.layers.PadLayer(incoming=incoming,
width=(scale, scale)),
num_filters=num_filters,
filter_size=[3, 3],
dilation=(scale, scale),
W=W.dimshuffle(1, 0, 2, 3) * theano.shared(
lasagne.utils.floatX(sinit.sample(num_filters)),
name + '_coeff_' + str(scale)).dimshuffle(
'x', 0, 'x', 'x'), #.dimshuffle('x',0),
b=None,
nonlinearity=None,
name=name + str(scale)))
return ESL(nd + [n])
def MDBLOCK(incoming, num_filters, scales, name, nonlinearity):
return NL(
BN(ESL([
incoming,
MDCL(
NL(
BN(MDCL(
NL(BN(incoming, name=name + 'bnorm0'), nonlinearity),
num_filters, scales, name),
name=name + 'bnorm1'), nonlinearity), num_filters,
scales, name + '2')
]),
name=name + 'bnorm2'), nonlinearity)
def ResDropNoPre(incoming, IB, p):
return NL(ESL([IfElseDropLayer(IB,survival_p=p),incoming]),elu)
def ResDrop(incoming, IB, p):
return ESL([IfElseDropLayer(IB,survival_p=p),incoming])
def ResLayer(incoming, IB):
return NL(ESL([IB,incoming]),elu)
def get_model(interp=False):
dims, n_channels = tuple(cfg['dims']), cfg['n_channels']
shape = (None, n_channels) + dims
l_in = lasagne.layers.InputLayer(shape=shape)
l_enc_conv1 = C2D(incoming=l_in,
num_filters=128,
filter_size=[5, 5],
stride=[2, 2],
pad=(2, 2),
W=initmethod(0.02),
nonlinearity=lrelu(0.2),
name='enc_conv1')
l_enc_conv2 = BN(C2D(incoming=l_enc_conv1,
num_filters=256,
filter_size=[5, 5],
stride=[2, 2],
pad=(2, 2),
W=initmethod(0.02),
nonlinearity=lrelu(0.2),
name='enc_conv2'),
name='bnorm2')
l_enc_conv3 = BN(C2D(incoming=l_enc_conv2,
num_filters=512,
filter_size=[5, 5],
stride=[2, 2],
pad=(2, 2),
W=initmethod(0.02),
nonlinearity=lrelu(0.2),
name='enc_conv3'),
name='bnorm3')
l_enc_conv4 = BN(C2D(incoming=l_enc_conv3,
num_filters=1024,
filter_size=[5, 5],
stride=[2, 2],
pad=(2, 2),
W=initmethod(0.02),
nonlinearity=lrelu(0.2),
name='enc_conv4'),
name='bnorm4')
print(lasagne.layers.get_output_shape(l_enc_conv4, (196, 3, 64, 64)))
l_enc_fc1 = BN(DL(incoming=l_enc_conv4,
num_units=1000,
W=initmethod(0.02),
nonlinearity=relu,
name='enc_fc1'),
name='bnorm_enc_fc1')
# Define latent values
l_enc_mu, l_enc_logsigma = [
BN(DL(incoming=l_enc_fc1,
num_units=cfg['num_latents'],
nonlinearity=None,
name='enc_mu'),
name='mu_bnorm'),
BN(DL(incoming=l_enc_fc1,
num_units=cfg['num_latents'],
nonlinearity=None,
name='enc_logsigma'),
name='ls_bnorm')
]
l_Z_IAF = GaussianSampleLayer(l_enc_mu, l_enc_logsigma, name='l_Z_IAF')
l_IAF_mu, l_IAF_logsigma = [
MADE(l_Z_IAF, [cfg['num_latents']], 'l_IAF_mu'),
MADE(l_Z_IAF, [cfg['num_latents']], 'l_IAF_ls')
]
l_Z = IAFLayer(l_Z_IAF, l_IAF_mu, l_IAF_logsigma, name='l_Z')
l_dec_fc2 = DL(incoming=l_Z,
num_units=512 * 16,
nonlinearity=lrelu(0.2),
W=initmethod(0.02),
name='l_dec_fc2')
l_unflatten = lasagne.layers.ReshapeLayer(
incoming=l_dec_fc2,
shape=([0], 512, 4, 4),
)
l_dec_conv1 = DeconvLayer(incoming=l_unflatten,
num_filters=512,
filter_size=[5, 5],
stride=[2, 2],
crop=(2, 2),
W=initmethod(0.02),
nonlinearity=None,
name='dec_conv1')
l_dec_conv2a = MDBLOCK(incoming=l_dec_conv1,
num_filters=512,
scales=[0, 2],
name='dec_conv2a',
nonlinearity=lrelu(0.2))
l_dec_conv2 = DeconvLayer(incoming=l_dec_conv2a,
num_filters=256,
filter_size=[5, 5],
stride=[2, 2],
crop=(2, 2),
W=initmethod(0.02),
nonlinearity=None,
name='dec_conv2')
l_dec_conv3a = MDBLOCK(incoming=l_dec_conv2,
num_filters=256,
scales=[0, 2, 3],
name='dec_conv3a',
nonlinearity=lrelu(0.2))
l_dec_conv3 = DeconvLayer(incoming=l_dec_conv3a,
num_filters=128,
filter_size=[5, 5],
stride=[2, 2],
crop=(2, 2),
W=initmethod(0.02),
nonlinearity=None,
name='dec_conv3')
l_dec_conv4a = MDBLOCK(incoming=l_dec_conv3,
num_filters=128,
scales=[0, 2, 3],
name='dec_conv4a',
nonlinearity=lrelu(0.2))
l_dec_conv4 = BN(DeconvLayer(incoming=l_dec_conv4a,
num_filters=128,
filter_size=[5, 5],
stride=[2, 2],
crop=(2, 2),
W=initmethod(0.02),
nonlinearity=lrelu(0.2),
name='dec_conv4'),
name='bnorm_dc4')
R = NL(MDCL(l_dec_conv4, num_filters=2, scales=[2, 3, 4], name='R'),
sigmoid)
G = NL(
ESL([
MDCL(l_dec_conv4, num_filters=2, scales=[2, 3, 4], name='G_a'),
MDCL(R, num_filters=2, scales=[2, 3, 4], name='G_b')
]), sigmoid)
B = NL(
ESL([
MDCL(l_dec_conv4, num_filters=2, scales=[2, 3, 4], name='B_a'),
MDCL(CL([R, G]), num_filters=2, scales=[2, 3, 4], name='B_b')
]), sigmoid)
l_out = CL([
beta_layer(SL(R, slice(0, 1), 1), SL(R, slice(1, 2), 1)),
beta_layer(SL(G, slice(0, 1), 1), SL(G, slice(1, 2), 1)),
beta_layer(SL(B, slice(0, 1), 1), SL(B, slice(1, 2), 1))
])
minibatch_discrim = MinibatchLayer(
lasagne.layers.GlobalPoolLayer(l_enc_conv4),
num_kernels=500,
name='minibatch_discrim')
l_discrim = DL(incoming=minibatch_discrim,
num_units=3,
nonlinearity=lasagne.nonlinearities.softmax,
b=None,
W=initmethod(0.02),
name='discrimi')
return {
'l_in': l_in,
'l_out': l_out,
'l_mu': l_enc_mu,
'l_ls': l_enc_logsigma,
'l_Z': l_Z,
'l_IAF_mu': l_IAF_mu,
'l_IAF_ls': l_IAF_logsigma,
'l_Z_IAF': l_Z_IAF,
'l_introspect': [l_enc_conv1, l_enc_conv2, l_enc_conv3, l_enc_conv4],
'l_discrim': l_discrim
}
def __init__(self,
z,
hidden_sizes,
name,
nonlinearity=lasagne.nonlinearities.rectify,
output_nonlinearity=None,
**kwargs):
# self.rng = rng if rng else RandomStreams(lasagne.random.get_rng().randint(1234))
super(MADE, self).__init__(z, **kwargs)
# Incoming latents
self.z = z
# List defining hidden units in each layer
self.hidden_sizes = hidden_sizes
# Layer name for saving parameters.
self.name = name
# nonlinearity
self.nonlinearity = nonlinearity
# Output nonlinearity
self.output_nonlinearity = output_nonlinearity
# Control parameters from original MADE
mask_distribution = 0
use_cond_mask = False
direct_input_connect = "Output"
direct_output_connect = False
self.shuffled_once = False
# Mask generator
self.mask_generator = MaskGenerator(
lasagne.layers.get_output_shape(z)[1], hidden_sizes,
mask_distribution)
# Build the MADE
# TODO: Consider making this more compact by directly writing to the layers list
self.input_layer = MaskedLayer(incoming=z,
num_units=hidden_sizes[0],
mask_generator=self.mask_generator,
layerIdx=0,
W=lasagne.init.Orthogonal('relu'),
nonlinearity=self.nonlinearity,
name=self.name + '_input')
self.layers = [self.input_layer]
for i in range(1, len(hidden_sizes)):
self.layers += [
MaskedLayer(incoming=self.layers[-1],
num_units=hidden_sizes[i],
mask_generator=self.mask_generator,
layerIdx=i,
W=lasagne.init.Orthogonal('relu'),
nonlinearity=self.nonlinearity,
name=self.name + '_layer_' + str(i))
]
outputLayerIdx = len(self.layers)
# Output layer
self.layers += [
MaskedLayer(incoming=self.layers[-1],
num_units=lasagne.layers.get_output_shape(z)[1],
mask_generator=self.mask_generator,
layerIdx=outputLayerIdx,
W=lasagne.init.Orthogonal('relu'),
nonlinearity=self.output_nonlinearity,
name=self.name + '_output_W'),
DIML(incoming=z,
num_units=lasagne.layers.get_output_shape(z)[1],
mask_generator=self.mask_generator,
layerIdx=outputLayerIdx,
W=lasagne.init.Orthogonal('relu'),
nonlinearity=self.output_nonlinearity,
name=self.name + '_output_D')
]
masks_updates = [
layer_mask_update for l in self.layers
for layer_mask_update in l.shuffle_update
]
self.update_masks = theano.function(name='update_masks',
inputs=[],
updates=masks_updates)
# Make the true output layer by ESL'ing the DIML and masked layer
self.final_layer = ESL([self.layers[-2], self.layers[-1]])
def MDCL(incoming, num_filters, scales, name, dnn=True):
if dnn:
from lasagne.layers.dnn import Conv2DDNNLayer as C2D
# W initialization method--this should also work as Orthogonal('relu'), but I have yet to validate that as thoroughly.
winit = initmethod(0.02)
# Initialization method for the coefficients
sinit = lasagne.init.Constant(1.0 / (1 + len(scales)))
# Number of incoming channels
ni = lasagne.layers.get_output_shape(incoming)[1]
# Weight parameter--the primary parameter for this block
W = theano.shared(lasagne.utils.floatX(
winit.sample((num_filters,
lasagne.layers.get_output_shape(incoming)[1], 3, 3))),
name=name + 'W')
# Primary Convolution Layer--No Dilation
n = C2D(
incoming=incoming,
num_filters=num_filters,
filter_size=[3, 3],
stride=[1, 1],
pad=(1, 1),
W=W * theano.shared(lasagne.utils.floatX(
sinit.sample(num_filters)), name + '_coeff_base').dimshuffle(
0, 'x', 'x', 'x'
), # Note the broadcasting dimshuffle for the num_filter scalars.
b=None,
nonlinearity=None,
name=name + 'base')
# List of remaining layers. This should probably just all be concatenated into a single list rather than being a separate deal.
nd = []
for i, scale in enumerate(scales):
# I don't think 0 dilation is technically defined (or if it is it's just the regular filter) but I use it here as a convenient keyword to grab the 1x1 mean conv.
if scale == 0:
nd.append(
C2D(incoming=incoming,
num_filters=num_filters,
filter_size=[1, 1],
stride=[1, 1],
pad=(0, 0),
W=T.mean(W, axis=[2, 3]).dimshuffle(0, 1, 'x', 'x') *
theano.shared(
lasagne.utils.floatX(sinit.sample(num_filters)),
name + '_coeff_1x1').dimshuffle(0, 'x', 'x', 'x'),
b=None,
nonlinearity=None,
name=name + str(scale)))
# Note the dimshuffles in this layer--these are critical as the current DilatedConv2D implementation uses a backward pass.
else:
nd.append(
lasagne.layers.DilatedConv2DLayer(
incoming=lasagne.layers.PadLayer(incoming=incoming,
width=(scale, scale)),
num_filters=num_filters,
filter_size=[3, 3],
dilation=(scale, scale),
W=W.dimshuffle(1, 0, 2, 3) * theano.shared(
lasagne.utils.floatX(sinit.sample(num_filters)), name +
'_coeff_' + str(scale)).dimshuffle('x', 0, 'x', 'x'),
b=None,
nonlinearity=None,
name=name + str(scale)))
return ESL(nd + [n])
def ResLayer(incoming, IB, nonlinearity):
return NL(ESL([IB, incoming]), nonlinearity)
def DSL(incoming, num_filters, scales, name, dnn=True):
if dnn:
from lasagne.layers.dnn import Conv2DDNNLayer as C2D
# W initialization method--this should also work as Orthogonal('relu'), but I have yet to validate that as thoroughly.
winit = initmethod(0.02)
# Initialization method for the coefficients
sinit = lasagne.init.Constant(1.0 / (1 + len(scales)))
# Number of incoming channels
ni = lasagne.layers.get_output_shape(incoming)[1]
# Weight parameter--the primary parameter for this block
W = theano.shared(lasagne.utils.floatX(
winit.sample((num_filters,
lasagne.layers.get_output_shape(incoming)[1], 3, 3))),
name=name + 'W')
# Main layer--3x3 conv with stride 2
n = C2D(incoming=incoming,
num_filters=num_filters,
filter_size=[3, 3],
stride=[2, 2],
pad=(1, 1),
W=W *
theano.shared(lasagne.utils.floatX(sinit.sample(num_filters)),
name + '_coeff_base').dimshuffle(0, 'x', 'x', 'x'),
b=None,
nonlinearity=None,
name=name + 'base')
nd = []
for i, scale in enumerate(scales):
p = P2D(
incoming=incoming,
pool_size=scale,
stride=2,
pad=(1, 1) if i else (0, 0),
mode='average_exc_pad',
)
nd.append(
C2D(
incoming=p,
num_filters=num_filters,
filter_size=[3, 3],
stride=(1, 1),
pad=(1, 1),
W=W *
theano.shared(lasagne.utils.floatX(sinit.sample(num_filters)),
name + '_coeff_' + str(scale)).dimshuffle(
0, 'x', 'x', 'x'), #.dimshuffle('x',0),
b=None,
nonlinearity=None,
name=name + str(scale)))
nd.append(
C2D(incoming=incoming,
num_filters=num_filters,
filter_size=[1, 1],
stride=[2, 2],
pad=(0, 0),
W=T.mean(W, axis=[2, 3]).dimshuffle(0, 1, 'x', 'x') *
theano.shared(lasagne.utils.floatX(sinit.sample(num_filters)),
name + '_coeff_1x1').dimshuffle(0, 'x', 'x', 'x'),
b=None,
nonlinearity=None,
name=name + '1x1'))
return ESL(nd + [n])
def USL(incoming, num_filters, scales, name, dnn=True):
if dnn:
from lasagne.layers.dnn import Conv2DDNNLayer as C2D
# W initialization method--this should also work as Orthogonal('relu'), but I have yet to validate that as thoroughly.
winit = initmethod(0.02)
# Initialization method for the coefficients
sinit = lasagne.init.Constant(1.0 / (1 + len(scales)))
# Number of incoming channels
ni = lasagne.layers.get_output_shape(incoming)[1]
# Weight parameter--the primary parameter for this block
W = theano.shared(lasagne.utils.floatX(
winit.sample((num_filters,
lasagne.layers.get_output_shape(incoming)[1], 3, 3))),
name=name + 'W')
# Primary Convolution Layer--No Dilation
n = C2D(incoming=Upscale2DLayer(incoming, 2),
num_filters=num_filters,
filter_size=[3, 3],
stride=[1, 1],
pad=(1, 1),
W=W *
theano.shared(lasagne.utils.floatX(sinit.sample(num_filters)),
name + '_coeff_base').dimshuffle(0, 'x', 'x', 'x'),
b=None,
nonlinearity=None,
name=name + 'base')
# Remaining layers
nd = []
for i, scale in enumerate(scales):
if scale == 0:
nd.append(
C2D(incoming=Upscale2DLayer(incoming, 2),
num_filters=num_filters,
filter_size=[1, 1],
stride=[1, 1],
pad=(0, 0),
W=T.mean(W, axis=[2, 3]).dimshuffle(0, 1, 'x', 'x') *
theano.shared(
lasagne.utils.floatX(sinit.sample(num_filters)),
name + '_coeff_1x1').dimshuffle(0, 'x', 'x', 'x'),
b=None,
nonlinearity=None,
name=name + '1x1'))
else:
nd.append(
lasagne.layers.DilatedConv2DLayer(
incoming=lasagne.layers.PadLayer(incoming=Upscale2DLayer(
incoming, 2),
width=(scale, scale)),
num_filters=num_filters,
filter_size=[3, 3],
dilation=(scale, scale),
W=W.dimshuffle(1, 0, 2, 3) * theano.shared(
lasagne.utils.floatX(sinit.sample(num_filters)), name +
'_coeff_' + str(scale)).dimshuffle('x', 0, 'x', 'x'),
b=None,
nonlinearity=None,
name=name + str(scale)))
# A single deconv layer is also concatenated here. Like I said, it's a prototype!
nd.append(
DeconvLayer(
incoming=incoming,
num_filters=num_filters,
filter_size=[3, 3],
stride=[2, 2],
crop=(1, 1),
W=W.dimshuffle(1, 0, 2, 3) *
theano.shared(lasagne.utils.floatX(sinit.sample(num_filters)),
name + '_coeff_deconv').dimshuffle('x', 0, 'x', 'x'),
b=None,
nonlinearity=None,
name=name + 'deconv'))
return ESL(nd + [n])
def DSL(incoming, num_filters, scales, name):
# Total number of layers
# W = theano.shared(lasagne.utils.floatX(Orthogonal(
winit = initmethod(0.02)
sinit = lasagne.init.Constant(1.0 / (1 + len(scales)))
# Number of incoming channels
ni = lasagne.layers.get_output_shape(incoming)[1]
# get weight parameter for this layer
W = theano.shared(lasagne.utils.floatX(
winit.sample((num_filters,
lasagne.layers.get_output_shape(incoming)[1], 3, 3))),
name=name + 'W')
n = C2D(incoming=incoming,
num_filters=num_filters,
filter_size=[3, 3],
stride=[2, 2],
pad=(1, 1),
W=W *
theano.shared(lasagne.utils.floatX(sinit.sample(num_filters)),
name + '_coeff_base').dimshuffle(0, 'x', 'x', 'x'),
b=None,
nonlinearity=None,
name=name + 'base')
# nc = [theano.shared(lasagne.utils.floatX(1.0/(1+len(scales))), name+'coeff_base')]
nd = []
for i, scale in enumerate(scales):
p = P2D(
incoming=incoming,
pool_size=scale,
stride=2,
pad=(1, 1) if i else (0, 0),
mode='average_exc_pad',
)
nd.append(
C2D(
incoming=p,
num_filters=num_filters,
filter_size=[3, 3],
stride=(1, 1),
pad=(1, 1),
W=W *
theano.shared(lasagne.utils.floatX(sinit.sample(num_filters)),
name + '_coeff_' + str(scale)).dimshuffle(
0, 'x', 'x', 'x'), #.dimshuffle('x',0),
b=None,
nonlinearity=None,
name=name + str(scale)))
nd.append(
C2D(incoming=incoming,
num_filters=num_filters,
filter_size=[1, 1],
stride=[2, 2],
pad=(0, 0),
W=T.mean(W, axis=[2, 3]).dimshuffle(0, 1, 'x', 'x') *
theano.shared(lasagne.utils.floatX(sinit.sample(num_filters)),
name + '_coeff_1x1').dimshuffle(0, 'x', 'x', 'x'),
b=None,
nonlinearity=None,
name=name + '1x1'))
# nc.append()
return ESL(nd + [n])