def deep2(input_var):
net = {}
net['data'] = InputLayer(inshape, input_var=input_var)
net['conv1'] = Conv2DLayer(net['data'],
num_filters=20,
filter_size=(1, 255),
stride=(1, 16),
pad='same',
nonlinearity=rectify)
net['conv2'] = Conv2DLayer(net['conv1'],
num_filters=20,
filter_size=(1, 255),
stride=(1, 16),
pad='same',
nonlinearity=rectify)
net['conv3'] = Conv2DLayer(
net['conv2'],
num_filters=20,
filter_size=(1, 127),
stride=(1, 16),
pad='same',
nonlinearity=rectify) # should have dims of (23 x 32 x ?)
net['conv4'] = Conv2DLayer(net['conv3'],
num_filters=20,
filter_size=(1, 15),
stride=1,
pad='same',
nonlinearity=rectify) # dims (23 x 32 x ?)
net['fcl'] = DenseLayer(net['conv4'], num_units=100, nonlinearity=rectify)
net['out'] = DenseLayer(net['fcl'],
num_units=outshape,
nonlinearity=sigmoid)
return net
def _build_network(self):
l_in = InputLayer(shape=(None, DIMS[0], DIMS[1], DIMS[2]),
name="input")
l_1 = Conv2DLayer(l_in,
num_filters=16,
filter_size=(8, 8),
stride=4,
nonlinearity=lasagne.nonlinearities.rectify,
name="conv1")
l_2 = Conv2DLayer(l_1,
num_filters=32,
filter_size=(4, 4),
stride=2,
nonlinearity=lasagne.nonlinearities.rectify,
name="conv2")
l_3 = batch_norm(
DenseLayer(l_2,
num_units=N_HIDDEN,
nonlinearity=lasagne.nonlinearities.rectify,
name="fc1"))
l_out = DenseLayer(l_3,
num_units=self.actionsNum,
nonlinearity=lasagne.nonlinearities.identity,
name="out")
return l_out, l_in.input_var
def conv_net(input_layer):
if self.n_mi_features != 0:
conv_input = SliceLayer(
input_layer,
indices=slice(0,
input_layer.shape[1] - self.n_mi_features))
mi_input = SliceLayer(
input_layer,
indices=slice(input_layer.shape[1] - self.n_mi_features,
None))
else:
conv_input = input_layer
mi_input = None
conv_input = ReshapeLayer(
conv_input, (-1, 1, self.input_size, self.input_size))
conv_layer_output_shapes = []
output = Conv2DLayer(conv_input, 64, 5, stride=2, pad='same')
conv_layer_output_shapes.append(output.output_shape[2])
output = Conv2DLayer(output, 128, 5, stride=2, pad='same')
conv_layer_output_shapes.append(output.output_shape[2])
output = ReshapeLayer(output, (-1, num_elems(output)))
if mi_input is not None:
output = ConcatLayer([output, mi_input], axis=1)
output = BatchNormLayer(DenseLayer(output, conv_output_size))
return output, conv_layer_output_shapes
def build_discriminator_32(image=None, ndf=128):
lrelu = LeakyRectify(0.2)
# input: images
InputImg = InputLayer(shape=(None, 3, 32, 32), input_var=image)
print("Dis Img_input:", InputImg.output_shape)
# Conv Layer
dis1 = Conv2DLayer(InputImg,
ndf, (4, 4), (2, 2),
pad=1,
W=Normal(0.02),
nonlinearity=lrelu)
print("Dis conv1:", dis1.output_shape)
# Conv Layer
dis2 = batch_norm(
Conv2DLayer(dis1,
ndf * 2, (4, 4), (2, 2),
pad=1,
W=Normal(0.02),
nonlinearity=lrelu))
print("Dis conv2:", dis2.output_shape)
# Conv Layer
dis3 = batch_norm(
Conv2DLayer(dis2,
ndf * 4, (4, 4), (2, 2),
pad=1,
W=Normal(0.02),
nonlinearity=lrelu))
print("Dis conv3:", dis3.output_shape)
# Conv Layer
dis4 = DenseLayer(dis3, 1, W=Normal(0.02), nonlinearity=sigmoid)
print("Dis output:", dis4.output_shape)
return dis4
def construct_vgg13net(channels=1, no_f_base=64, f_size=3, dropout=False, bs=None, num_classes=3, pad=1,nonlinearity=lasagne.nonlinearities.rectify, input_dim=[512,512], flip=True, fcn=4096):
net = {}
net["input"] = InputLayer(shape=(bs, channels, input_dim[0], input_dim[1]))
net["conv_11"] = Conv2DLayer(net["input"], no_f_base, f_size, pad=pad, flip_filters=flip, nonlinearity=nonlinearity)
net["conv_12"] = Conv2DLayer(net["conv_11"], no_f_base, f_size, pad=pad, flip_filters=flip, nonlinearity=nonlinearity)
net["pool1"] = Pool2DLayer(net["conv_12"], 2)
net["conv_21"] = Conv2DLayer(net["pool1"], no_f_base*2, f_size, pad=pad, flip_filters=flip, nonlinearity=nonlinearity)
net["conv_22"] = Conv2DLayer(net["conv_21"], no_f_base*2, f_size, pad=pad, flip_filters=flip, nonlinearity=nonlinearity)
net["pool2"] = Pool2DLayer(net["conv_22"], 2)
net["conv_31"] = Conv2DLayer(net["pool2"], no_f_base*4, f_size, pad=pad, flip_filters=flip, nonlinearity=nonlinearity)
net["conv_32"] = Conv2DLayer(net["conv_31"], no_f_base*4, f_size, pad=pad, flip_filters=flip, nonlinearity=nonlinearity)
net["conv_33"] = Conv2DLayer(net["conv_32"], no_f_base*4, f_size, pad=pad, flip_filters=flip, nonlinearity=nonlinearity)
net["pool3"] = Pool2DLayer(net["conv_33"], 2)
net["conv_41"] = Conv2DLayer(net["pool3"], no_f_base*8, f_size, pad=pad, flip_filters=flip, nonlinearity=nonlinearity)
net["conv_42"] = Conv2DLayer(net["conv_41"], no_f_base*8, f_size, pad=pad, flip_filters=flip, nonlinearity=nonlinearity)
net["conv_43"] = Conv2DLayer(net["conv_42"], no_f_base*8, f_size, pad=pad, flip_filters=flip, nonlinearity=nonlinearity)
net["pool4"] = Pool2DLayer(net["conv_43"], 2)
drop1 = net["full_con1"] = DenseLayer(net["pool4"], num_units=fcn, nonlinearity=nonlinearity)
if dropout:
drop1 = DropoutLayer(drop1, p=0.5)
drop2 = net["full_con2"] = DenseLayer(drop1, num_units=fcn, nonlinearity=nonlinearity)
if dropout:
drop2 = DropoutLayer(drop2, p=0.5)
net["full_con3"] = DenseLayer(drop2, num_units=num_classes, nonlinearity=None)
net["out"] = NonlinearityLayer(net["full_con3"], nonlinearity=lasagne.nonlinearities.softmax)
return net
del net
class net:
inp = InputLayer(self.input_shape)
# [None, 1, channels, time]
resh = ReshapeLayer(inp, [-1, 1] + list(self.input_shape[1:]))
conv1 = Conv2DLayer(resh, self.n_base_filter, 3, stride=(1, 2), pad="same")
gate1 = GatedConv2DLayer(conv1)
conv2 = Conv2DLayer(gate1, self.n_base_filter*2, 3, stride=(2, 2), pad="same")
norm1 = self.norm_func(conv2)
gate2 = GatedConv2DLayer(norm1)
conv3 = Conv2DLayer(gate2, self.n_base_filter*4, 3, stride=(2, 2), pad="same")
norm2 = self.norm_func(conv3)
gate3 = GatedConv2DLayer(norm2)
conv4 = Conv2DLayer(gate3, self.n_base_filter*8, (6, 3), stride=(1, 2), pad="same")
norm3 = self.norm_func(conv4)
gate4 = GatedConv2DLayer(norm3)
nonlinearity = None
if not self.wasserstein:
nonlinearity = T.nnet.sigmoid
out = DenseLayer(gate4, 1, nonlinearity=nonlinearity)
def build_critic(gan, input_var=None, do_batch_norm=False):
from lasagne.layers import (InputLayer, Conv2DLayer, DenseLayer)
from lasagne.nonlinearities import LeakyRectify
lrelu = LeakyRectify(0.2)
# input: (None, 1, 28, 28)
layer = InputLayer(shape=(None, 1, 28, 28), input_var=input_var)
# two convolutions
layer = BatchNorm(
Conv2DLayer(layer, 64, 5, stride=2, pad='same', nonlinearity=lrelu),
do_batch_norm)
layer = BatchNorm(
Conv2DLayer(layer, 128, 5, stride=2, pad='same', nonlinearity=lrelu),
do_batch_norm)
# fully-connected layer
layer = BatchNorm(DenseLayer(layer, 1024, nonlinearity=lrelu),
do_batch_norm)
# output layer
if gan in ('wgan', 'wgan-gp'):
layer = DenseLayer(layer, 1, nonlinearity=None, b=None)
elif gan in ('lsgan', ):
layer = DenseLayer(layer, 1, nonlinearity=None)
elif gan in ('dcgan', ):
from lasagne.nonlinearities import sigmoid
layer = DenseLayer(layer, 1, nonlinearity=sigmoid)
else:
raise Exception("GAN {} is not supported".format(gan))
print("Critic output: ", layer.output_shape)
return layer
def build_critic(input_var=None, model_name='wgan'):
from lasagne.layers import (InputLayer, Conv2DLayer, ReshapeLayer,
DenseLayer)
try:
from lasagne.layers.dnn import batch_norm_dnn as batch_norm
except ImportError:
from lasagne.layers import batch_norm
from lasagne.nonlinearities import LeakyRectify, sigmoid
lrelu = LeakyRectify(0.2)
# input: (None, 1, 28, 28)
layer = InputLayer(shape=(None, 1, 28, 28), input_var=input_var)
# two convolutions
layer = batch_norm(
Conv2DLayer(layer, 64, 5, stride=2, pad='same', nonlinearity=lrelu))
layer = batch_norm(
Conv2DLayer(layer, 128, 5, stride=2, pad='same', nonlinearity=lrelu))
# fully-connected layer
layer = batch_norm(DenseLayer(layer, 1024, nonlinearity=lrelu))
# output layer
if model_name == 'dcgan':
layer = DenseLayer(layer, 1, nonlinearity=sigmoid)
elif model_name == 'wgan':
layer = DenseLayer(layer, 1, nonlinearity=None, b=None)
elif model_name == 'lsgan':
layer = DenseLayer(layer, 1, nonlinearity=None)
print("critic output:", layer.output_shape)
return layer
def architecture(input_var, input_shape):
network_trained = {}
network_trained['input'] = InputLayer(input_shape, input_var)
kwargs = dict(nonlinearity=lasagne.nonlinearities.leaky_rectify,
W=lasagne.init.Orthogonal())
network_trained['conv1'] = Conv2DLayer(network_trained['input'], 64, 3,
**kwargs)
network_trained['conv2'] = Conv2DLayer(network_trained['conv1'], 32, 3,
**kwargs)
network_trained['mp3'] = MaxPool2DLayer(network_trained['conv2'], 3)
network_trained['conv4'] = Conv2DLayer(network_trained['mp3'], 128, 3,
**kwargs)
network_trained['conv5'] = Conv2DLayer(network_trained['conv4'], 64, 3,
**kwargs)
network_trained['mp6'] = MaxPool2DLayer(network_trained['conv5'], 3)
network_trained['fc7'] = DenseLayer(dropout(network_trained['mp6'], 0.5),
256, **kwargs)
network_trained['fc8'] = DenseLayer(dropout(network_trained['fc7'], 0.5),
64, **kwargs)
network_trained['fc9'] = DenseLayer(
dropout(network_trained['fc8'], 0.5),
1,
nonlinearity=lasagne.nonlinearities.sigmoid,
W=lasagne.init.Orthogonal())
return network_trained
def __init__(self,
incoming,
filter_size=1,
pad="same",
nonlinearity=None,
**kwargs):
"""
Creates a GatedConv1DLayer instance
:param incoming: incoming layer
:param num_filters: int, number of conv filters
:param filter_size: int, size of conv filters
:param nonlinearity: activation function for h
"""
super(GatedConv2DLayer, self).__init__(incoming, **kwargs)
num_filters = incoming.output_shape[1]
h = Conv2DLayer(incoming,
num_filters,
filter_size,
pad=pad,
nonlinearity=nonlinearity,
**kwargs)
g = Conv2DLayer(incoming,
num_filters,
filter_size,
pad=pad,
nonlinearity=T.nnet.sigmoid,
**kwargs)
self.h = get_output(h)
self.g = get_output(g)
self.get_output_shape_for = g.get_output_shape_for
def lasagne_model():
l_in = InputLayer(shape=(None, 1, 28, 28))
l_conv1 = Conv2DLayer(l_in,
num_filters=128,
filter_size=(3, 3),
nonlinearity=rectify)
l_conv1b = Conv2DLayer(l_conv1,
num_filters=128,
filter_size=(3, 3),
nonlinearity=rectify)
l_pool1 = MaxPool2DLayer(l_conv1b, pool_size=(2, 2))
l_dropout1 = DropoutLayer(l_pool1, p=0.2)
l_conv2 = Conv2DLayer(l_pool1,
num_filters=256,
filter_size=(3, 3),
nonlinearity=rectify)
l_conv2b = Conv2DLayer(l_conv2,
num_filters=256,
filter_size=(3, 3),
nonlinearity=rectify)
l_pool2 = MaxPool2DLayer(l_conv2b, pool_size=(2, 2))
l_dropout2 = DropoutLayer(l_pool2, p=0.2)
l_hidden3 = DenseLayer(l_dropout2, num_units=1024, nonlinearity=rectify)
l_dropout3 = DropoutLayer(l_hidden3, p=0.5)
l_hidden4 = DenseLayer(l_dropout3, num_units=1024, nonlinearity=rectify)
l_dropout4 = DropoutLayer(l_hidden4, p=0.5)
l_out = DenseLayer(l_dropout4, num_units=10, nonlinearity=softmax)
return l_out
def contraction(depth, deepest):
n_filters = filter_for_depth(depth)
incoming = net['input'] if depth == 0 else net['pool{}'.format(depth -
1)]
net['conv{}_1'.format(depth)] = Conv2DLayer(incoming,
num_filters=n_filters,
filter_size=3,
pad='valid',
W=HeNormal(gain='relu'),
nonlinearity=nonlinearity)
net['conv{}_2'.format(depth)] = Conv2DLayer(
net['conv{}_1'.format(depth)],
num_filters=n_filters,
filter_size=3,
pad='valid',
W=HeNormal(gain='relu'),
nonlinearity=nonlinearity)
if P.BATCH_NORMALIZATION:
net['conv{}_2'.format(depth)] = batch_norm(
net['conv{}_2'.format(depth)],
alpha=P.BATCH_NORMALIZATION_ALPHA)
if not deepest:
net['pool{}'.format(depth)] = MaxPool2DLayer(
net['conv{}_2'.format(depth)], pool_size=2, stride=2)
def q_network(state):
input_state = InputLayer(input_var=state,
shape=(None, self.state_dimension[0],
self.state_dimension[1],
self.state_dimension[2]))
input_state = DimshuffleLayer(input_state, pattern=(0, 3, 1, 2))
conv = Conv2DLayer(input_state,
num_filters=32,
filter_size=(8, 8),
stride=(4, 4),
nonlinearity=rectify)
conv = Conv2DLayer(conv,
num_filters=64,
filter_size=(4, 4),
stride=(2, 2),
nonlinearity=rectify)
conv = Conv2DLayer(conv,
num_filters=64,
filter_size=(3, 3),
stride=(1, 1),
nonlinearity=rectify)
flatten = FlattenLayer(conv)
dense = DenseLayer(flatten, num_units=512, nonlinearity=rectify)
q_values = DenseLayer(dense,
num_units=self.action_dimension,
nonlinearity=linear)
return q_values
def build_encoder_conv2d_32_hidden(self, l_input):
from lasagne.nonlinearities import sigmoid
from lasagne.nonlinearities import LeakyRectify
from lasagne.layers import Conv2DLayer
from lasagne.layers import InputLayer, ReshapeLayer, DenseLayer
try:
from lasagne.layers.dnn import batch_norm_dnn as batch_norm
except ImportError:
from lasagne.layers import batch_norm
# input: 3x28x28dim
lrelu = LeakyRectify(0.2)
layer = batch_norm(
Conv2DLayer(l_input,
64,
5,
stride=2,
pad='same',
nonlinearity=lrelu)) # original with relu
layer = batch_norm(
Conv2DLayer(layer,
128,
5,
stride=2,
pad='same',
nonlinearity=lrelu)) # original with relu
return ReshapeLayer(layer, ([0], 6272))
def _build_network(self):
l_in = InputLayer(self.input_shape, name="input")
l_1 = Conv2DLayer(l_in,
num_filters=32,
filter_size=(8, 8),
stride=4,
nonlinearity=lasagne.nonlinearities.rectify,
name="conv1")
l_2 = Conv2DLayer(l_1,
num_filters=64,
filter_size=(4, 4),
stride=2,
nonlinearity=lasagne.nonlinearities.rectify,
name="conv2")
l_3 = Conv2DLayer(l_2,
num_filters=64,
filter_size=(3, 3),
stride=1,
nonlinearity=lasagne.nonlinearities.rectify,
name="conv3")
l_4 = DenseLayer(l_3,
num_units=512,
nonlinearity=lasagne.nonlinearities.rectify,
name="fc1")
l_out = DenseLayer(l_4,
num_units=self.num_actions,
nonlinearity=lasagne.nonlinearities.identity,
W=lasagne.init.Normal(),
name="out")
return l_out, l_in.input_var
def build_model(batch_size=128):
x = T.tensor4('input')
layer = InputLayer((batch_size, 3, image_sz, image_sz), input_var=x)
conv1 = Conv2DLayer(layer, 64, 7, stride=2, pad='same')
pool1 = MaxPool2DLayer(conv1, 3, stride=2, pad=1)
conv2 = Conv2DLayer(pool1, 64, 1, pad='same')
conv3 = Conv2DLayer(conv2, 192, 3, pad='same')
pool3 = MaxPool2DLayer(conv3, 3, stride=2, pad=1)
incept3a = _inception(pool3, 64, 96, 128, 16, 32, 32)
incept3b = _inception(incept3a, 128, 128, 192, 32, 96, 64)
pool4 = MaxPool2DLayer(incept3b, 3, stride=2, pad=1)
incept4a = _inception(pool4, 192, 96, 208, 16, 48, 64)
incept4b = _inception(incept4a, 160, 112, 224, 24, 64, 64)
incept4c = _inception(incept4b, 128, 128, 256, 24, 64, 64)
incept4d = _inception(incept4c, 112, 144, 288, 32, 64, 64)
incept4e = _inception(incept4d, 256, 160, 320, 32, 128, 128)
pool5 = MaxPool2DLayer(incept4e, 3, stride=2, pad=1)
incept5a = _inception(pool5, 256, 160, 320, 32, 128, 128)
incept5b = _inception(incept5a, 384, 192, 384, 48, 128, 128)
pool6 = Pool2DLayer(incept5b, 7, stride=1, mode='average_exc_pad')
layer = DenseLayer(pool6, 1024)
layer = DenseLayer(layer, 1000, nonlinearity=None)
return layer, x
def add_layer(incoming, num_channels, dropout):
layer = ScaleLayer(incoming)
layer = BiasLayer(layer)
# Bottleneck layer to reduce number of input channels to 4 times the number of output channels
layer = NonlinearityLayer(layer, nonlinearity=rectify)
layer = Conv2DLayer(layer,
num_filters=4 * num_channels,
filter_size=(1, 1),
stride=(1, 1),
W=HeNormal(gain='relu'), b=None,
flip_filters=False,
nonlinearity=None)
layer = BatchNormLayer(layer, beta=None, gamma=None)
if dropout > 0:
layer = DropoutLayer(layer, p=dropout)
# Convolutional layer (using padding to keep same dimensions)
layer = NonlinearityLayer(layer, nonlinearity=rectify)
layer = Conv2DLayer(layer,
num_filters=num_channels,
filter_size=(3, 3),
stride=(1, 1),
W=HeNormal(gain='relu'), b=None,
pad='same',
flip_filters=False,
nonlinearity=None)
layer = BatchNormLayer(layer, beta=None, gamma=None)
if dropout > 0:
layer = DropoutLayer(layer, p=dropout)
# Concatenate the input filters with the new filters
layer = ConcatLayer([incoming, layer], axis=1)
return layer
def rnn_orig(input_var, seq_len, sz=51):
def add_shapes(sh1, sh2, axis=2):
if isinstance(sh2, tuple):
return sh1[:axis] + (sh1[axis] + sh2[axis], ) + sh1[axis + 1:]
else:
return sh1[:axis] + (sh1[axis] + sh2, ) + sh1[axis + 1:]
ret = {}
ret['input'] = in_layer = InputLayer((None, seq_len, 2, sz, sz), input_var)
ret['in_to_hid'] = in_to_hid = Conv2DLayer(InputLayer((None, 2, sz, sz)),
16,
7,
pad=3,
nonlinearity=sigmoid)
ret['post_concat'] = post_concat = Conv2DLayer(InputLayer(
add_shapes(in_to_hid.output_shape, 32, 1)),
32,
7,
pad=3,
nonlinearity=sigmoid)
ret['hid_to_hid'] = hid_to_hid = NonlinearityLayer(InputLayer(
post_concat.output_shape),
nonlinearity=None)
ret['rec'] = f = crl.ConcatRecurrentLayer(in_layer, in_to_hid, hid_to_hid,
post_concat)
ret['rec_resh'] = f = ReshapeLayer(f, (-1, [2], [3], [4]))
ret['y_pre'] = f = Conv2DLayer(f, 1, 7, pad=3, nonlinearity=sigmoid)
ret['output'] = f = ReshapeLayer(f, (-1, seq_len, [1], [2], [3]))
return ret, nn.layers.get_output(ret['output']), nn.layers.get_output(
ret['output'], deterministic=True)
def build_architecture(input_shape, trained_weights=None):
""" Build the Theano symbolic graph representing the CNN model.
:param input_shape: A tuple representing the input shape (h,w)
:param trained_weights: Pre-trained weights. If None, the network is initialized at random.
:return: A dictionary containing all layers
"""
net = {}
net['input'] = InputLayer(input_shape)
net['conv1'] = batch_norm(Conv2DLayer(net['input'], num_filters=96, filter_size=11, stride=4, flip_filters=False))
net['pool1'] = MaxPool2DLayer(net['conv1'], pool_size=3, stride=1)
net['conv2'] = batch_norm(Conv2DLayer(net['pool1'], num_filters=256, filter_size=5, pad=2, flip_filters=False))
net['pool2'] = MaxPool2DLayer(net['conv2'], pool_size=3, stride=4)
net['conv3'] = batch_norm(Conv2DLayer(net['pool2'], num_filters=384, filter_size=3, pad=1, flip_filters=False))
net['conv4'] = batch_norm(Conv2DLayer(net['conv3'], num_filters=384, filter_size=3, pad=1, flip_filters=False))
net['conv5'] = batch_norm(Conv2DLayer(net['conv4'], num_filters=256, filter_size=3, pad=1, flip_filters=False))
net['pool5'] = MaxPool2DLayer(net['conv5'], pool_size=3, stride=6)
net['fc1'] = batch_norm(DenseLayer(net['pool5'], num_units=2048))
net['fc2'] = batch_norm(DenseLayer(net['fc1'], num_units=2048))
if trained_weights:
lasagne.layers.set_all_param_values(net['fc2'], trained_weights)
return net
def build_critic(input_var=None):
from lasagne.layers import (InputLayer, Conv2DLayer, ReshapeLayer,
DenseLayer)
try:
from lasagne.layers.dnn import batch_norm_dnn as batch_norm
except ImportError:
from lasagne.layers import batch_norm
from lasagne.nonlinearities import LeakyRectify
lrelu = LeakyRectify(0.2)
# input: (None, 3, 64, 64)
layer = InputLayer(shape=(None, 3, 64, 64), input_var=input_var)
layer = GAN.GaussianNoiseLayer(layer, sigma=0.5)
# two convolutions
layer = batch_norm(
Conv2DLayer(layer, 64, 5, stride=2, pad='same', nonlinearity=lrelu))
layer = batch_norm(
Conv2DLayer(layer, 128, 5, stride=2, pad='same', nonlinearity=lrelu))
layer = batch_norm(
Conv2DLayer(layer, 256, 5, stride=2, pad='same', nonlinearity=lrelu))
layer = batch_norm(
Conv2DLayer(layer, 512, 5, stride=2, pad='same', nonlinearity=lrelu))
# fully-connected layer
layer = batch_norm(DenseLayer(layer, 1024, nonlinearity=lrelu))
# output layer (linear and without bias)
layer = DenseLayer(layer, 1, nonlinearity=None, b=None)
print("critic output:", layer.output_shape)
return layer
def build_nature_network_lstm(self, input_width, input_height, output_dim,
num_frames, batch_size):
"""
Build a large network consistent with the DeepMind Nature paper.
"""
from lasagne.layers import Conv2DLayer
l_in = lasagne.layers.InputLayer(shape=(batch_size, num_frames,
input_width, input_height))
l_in_reshape = lasagne.layers.ReshapeLayer(
l_in, (batch_size * num_frames, 1, input_width, input_height))
l_conv1 = Conv2DLayer(l_in_reshape,
num_filters=32,
filter_size=(8, 8),
stride=(4, 4),
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.HeUniform(),
b=lasagne.init.Constant(.1))
l_conv2 = Conv2DLayer(l_conv1,
num_filters=64,
filter_size=(4, 4),
stride=(2, 2),
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.HeUniform(),
b=lasagne.init.Constant(.1))
l_conv3 = Conv2DLayer(l_conv2,
num_filters=64,
filter_size=(3, 3),
stride=(1, 1),
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.HeUniform(),
b=lasagne.init.Constant(.1))
l_hidden1 = lasagne.layers.DenseLayer(
l_conv3,
num_units=512,
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.HeUniform(),
b=lasagne.init.Constant(.1))
#l_out = lasagne.layers.DenseLayer(
# l_hidden1,
# num_units=output_dim,
# nonlinearity=None,
# W=lasagne.init.HeUniform(),
# b=lasagne.init.Constant(.1)
#)
l_f = lasagne.layers.FlattenLayer(l_hidden1)
l_i = lasagne.layers.ReshapeLayer(l_f, (-1, num_frames, [1]))
l_lstm1 = lasagne.layers.LSTMLayer(l_i, 512, grad_clipping=100)
l_out = lasagne.layers.DenseLayer(l_lstm1,
num_units=output_dim,
nonlinearity=None,
W=lasagne.init.HeUniform(),
b=lasagne.init.Constant(.1))
return l_out
def residual_block(l, increase_dim=False, projection=True):
input_num_filters = l.output_shape[1]
if increase_dim:
first_stride = (2, 2)
out_num_filters = input_num_filters * 2
else:
first_stride = (1, 1)
out_num_filters = input_num_filters
stack_1 = batch_norm(
Conv2DLayer(l,
num_filters=out_num_filters,
filter_size=(3, 3),
stride=first_stride,
nonlinearity=rectify,
pad='same',
W=lasagne.init.HeNormal(gain='relu'),
flip_filters=False))
stack_2 = batch_norm(
Conv2DLayer(stack_1,
num_filters=out_num_filters,
filter_size=(3, 3),
stride=(1, 1),
nonlinearity=None,
pad='same',
W=lasagne.init.HeNormal(gain='relu'),
flip_filters=False))
# add shortcut connections
if increase_dim:
if projection:
# projection shortcut, as option B in paper
projection = batch_norm(
Conv2DLayer(l,
num_filters=out_num_filters,
filter_size=(1, 1),
stride=(2, 2),
nonlinearity=None,
pad='same',
b=None,
flip_filters=False))
block = NonlinearityLayer(ElemwiseSumLayer(
[stack_2, projection]),
nonlinearity=rectify)
else:
# identity shortcut, as option A in paper
identity = ExpressionLayer(
l, lambda X: X[:, :, ::2, ::2], lambda s:
(s[0], s[1], s[2] // 2, s[3] // 2))
padding = PadLayer(identity, [out_num_filters // 4, 0, 0],
batch_ndim=1)
block = NonlinearityLayer(ElemwiseSumLayer([stack_2, padding]),
nonlinearity=rectify)
else:
block = NonlinearityLayer(ElemwiseSumLayer([stack_2, l]),
nonlinearity=rectify)
return block
def architecture_upconv_conv2(input_var, input_shape, n_conv_layers,
n_conv_filters):
net = {}
kwargs = dict(nonlinearity=lasagne.nonlinearities.elu,
W=lasagne.init.HeNormal())
net['data'] = InputLayer(input_shape, input_var)
print("\rLayer output shapes")
print(net['data'].output_shape)
# Bunch of 3 x 3 convolution layers: experimentally we found that, adding 3 conv layers in start than in middle is better: but why?
i = 'data'
j = 'c1'
for idx in range(n_conv_layers):
print("Conv layer index: %d" % (idx + 1))
net[j] = batch_norm(
Conv2DLayer(net[i],
num_filters=n_conv_filters,
filter_size=3,
stride=1,
pad=1,
**kwargs))
print(net[j].output_shape)
# renaming for next iteration
i = j
j = j[:-1] + str(idx + 2)
# Bunch of transposed convolution layers
'''net['uc1'] = batch_norm(TransposedConv2DLayer(net[i], num_filters= 1, filter_size= 4, stride = 2, crop=1, **kwargs))
print(net['uc1'].output_shape)
net['uc2'] = batch_norm(TransposedConv2DLayer(net['uc1'], num_filters= 1, filter_size= 4, stride = 2, crop=1, **kwargs))
print(net['uc2'].output_shape)'''
# slicing the output to 115 x 80 size
'''net['s1'] = lasagne.layers.SliceLayer(net['uc2'], slice(0, 115), axis=-2)
print(net['s1'].output_shape)
net['out'] = lasagne.layers.SliceLayer(net['s1'], slice(0, 80), axis=-1)'''
net['c1'] = batch_norm(
Conv2DLayer(net[i],
num_filters=1,
filter_size=3,
stride=1,
pad=1,
**kwargs))
print(net['c1'].output_shape)
net['out'] = lasagne.layers.PadLayer(net['c1'], width=2)
print(net['out'].output_shape)
print("Number of parameter to be learned: %d" %
(lasagne.layers.count_params(net['out'])))
return net['out']
def indiv_block(incoming,num_filt): ''' Returns the conv+concat+bn block network ''' conv_a = Conv2DLayer(incoming,num_filters=num_filt, filter_size=(3,3), pad='same', W = lasagne.init.GlorotUniform()) # Default non-linearity of lasagne's Conv2DLayer is rectify. conv_b = Conv2DLayer(conv_a,num_filters=num_filt, filter_size=(3,3), pad='same', W = lasagne.init.GlorotUniform()) conv_concat = ConcatLayer([conv_a, conv_b]) incoming = BatchNormLayer(conv_concat) return incoming
def conv_activation_bn(input_layer, name = '', pad='same', activation = 'relu', W_init = -1, use_bn = True, **kwargs):
if use_bn:
conv = Conv2DLayer(input_layer, name = name+'_linear', nonlinearity=linear, pad=pad,
flip_filters=False, W = W_init, b = Constant(0.), **kwargs)
bn = BatchNormLayer(conv, name = name+'_bn')
out = NonlinearityLayer(bn, name = name+'_activation', nonlinearity = activation)
else:
out = Conv2DLayer(input_layer, name = name, nonlinearity=activation, pad=pad,
flip_filters=False, W = W_init, b = Constant(0.), **kwargs)
return out
def build_network(input_var, num_input_channels, num_classes):
conv_defs = {
'W': lasagne.init.HeNormal('relu'),
'b': lasagne.init.Constant(0.0),
'filter_size': (3, 3),
'stride': (1, 1),
'nonlinearity': lasagne.nonlinearities.LeakyRectify(0.1)
}
nin_defs = {
'W': lasagne.init.HeNormal('relu'),
'b': lasagne.init.Constant(0.0),
'nonlinearity': lasagne.nonlinearities.LeakyRectify(0.1)
}
dense_defs = {
'W': lasagne.init.HeNormal(1.0),
'b': lasagne.init.Constant(0.0),
'nonlinearity': lasagne.nonlinearities.softmax
}
wn_defs = {
'momentum': config.batch_normalization_momentum
}
net = InputLayer ( name='input', shape=(None, num_input_channels, 28, 28), input_var=input_var)
net = GaussianNoiseLayer(net, name='noise', sigma=config.augment_noise_stddev)
net = WN(Conv2DLayer (net, name='conv1a', num_filters=32, pad='same', **conv_defs), **wn_defs)
net = WN(Conv2DLayer (net, name='conv1b', num_filters=64, pad='same', **conv_defs), **wn_defs)
# net = WN(Conv2DLayer (net, name='conv1c', num_filters=128, pad='same', **conv_defs), **wn_defs)
net = MaxPool2DLayer (net, name='pool1', pool_size=(2, 2))
net = DropoutLayer (net, name='drop1', p=.5)
net = WN(Conv2DLayer (net, name='conv2a', num_filters=32, pad='same', **conv_defs), **wn_defs)
net = WN(Conv2DLayer (net, name='conv2b', num_filters=64, pad='same', **conv_defs), **wn_defs)
# net = WN(Conv2DLayer (net, name='conv2c', num_filters=256, pad='same', **conv_defs), **wn_defs)
net = MaxPool2DLayer (net, name='pool2', pool_size=(2, 2))
net = DropoutLayer (net, name='drop2', p=.5)
net = WN(Conv2DLayer (net, name='conv3a', num_filters=32, pad=0, **conv_defs), **wn_defs)
# net = WN(NINLayer (net, name='conv3b', num_units=256, **nin_defs), **wn_defs)
net = WN(NINLayer (net, name='conv3c', num_units=256, **nin_defs), **wn_defs)
net = GlobalPoolLayer (net, name='pool3')
net = WN(DenseLayer (net, name='dense', num_units=num_classes, **dense_defs), **wn_defs)
# net = GaussianNoiseLayer(net, name='noise', sigma=config.augment_noise_stddev)
# net = WN(DenseLayer (net, name='dense1', num_units=256, **dense_defs), **wn_defs)
# net = DropoutLayer (net, name='drop1', p=.5)
# net = WN(DenseLayer (net, name='dense2', num_units=256, **dense_defs), **wn_defs)
# net = DropoutLayer (net, name='drop2', p=.5)
# net = WN(DenseLayer (net, name='dense3', num_units=256, **dense_defs), **wn_defs)
#
# net = WN(DenseLayer (net, name='dense4', num_units=num_classes, **dense_defs), **wn_defs)
return net
def G_mnist_mode_recovery(
num_channels = 1,
resolution = 32,
fmap_base = 64,
fmap_decay = 1.0,
fmap_max = 256,
latent_size = None,
label_size = 10,
normalize_latents = True,
use_wscale = False,
use_pixelnorm = False,
use_batchnorm = True,
tanh_at_end = True,
progressive = False,
**kwargs):
R = int(np.log2(resolution))
assert resolution == 2**R and resolution >= 4
cur_lod = theano.shared(np.float32(0.0))
def nf(stage): return min(int(fmap_base / (2.0 ** (stage * fmap_decay))), fmap_max)
def PN(layer): return PixelNormLayer(layer, name=layer.name+'pn') if use_pixelnorm else layer
def BN(layer): return lasagne.layers.batch_norm(layer) if use_batchnorm else layer
def WS(layer): return WScaleLayer(layer, name=layer.name+'S') if use_wscale else layer
if latent_size is None: latent_size = nf(0)
input_layers = [InputLayer(name='Glatents', shape=[None, latent_size])]
net = input_layers[-1]
if normalize_latents:
net = PixelNormLayer(net, name='Glnorm')
if label_size:
input_layers += [InputLayer(name='Glabels', shape=[None, label_size])]
net = ConcatLayer (name='Gina', incomings=[net, input_layers[-1]])
net = ReshapeLayer(name='Ginb', incoming=net, shape=[[0], [1], 1, 1])
net = PN(BN(WS(Conv2DLayer(net, name='G1a', num_filters=64, filter_size=4, pad='full', nonlinearity=vlrelu, W=irelu))))
lods = [net]
for I in xrange(2, R): # I = 2, 3, ..., R-1
net = Upscale2DLayer(net, name='G%dup' % I, scale_factor=2)
net = PN(BN(WS(Conv2DLayer(net, name='G%da' % I, num_filters=nf(I-1), filter_size=3, pad=1, nonlinearity=vlrelu, W=irelu))))
lods += [net]
if progressive:
lods = [WS(Conv2DLayer(l, name='Glod%d' % i, num_filters=num_channels, filter_size=3, pad=1, nonlinearity=linear, W=ilinear)) for i, l in enumerate(reversed(lods))] # Should be this
#lods = [WS(NINLayer(l, name='Glod%d' % i, num_units=num_channels, nonlinearity=linear, W=ilinear)) for i, l in enumerate(reversed(lods))] # .. but this is better
output_layer = LODSelectLayer(name='Glod', incomings=lods, cur_lod=cur_lod, first_incoming_lod=0)
else:
net = WS(Conv2DLayer(net, name='toRGB', num_filters=num_channels, filter_size=3, pad=1, nonlinearity=linear, W=ilinear)) # Should be this
#net = WS(NINLayer(net, name='toRGB', num_units=num_channels, nonlinearity=linear, W=ilinear)) # .. but this is better
output_layer = net
if tanh_at_end:
output_layer = NonlinearityLayer(output_layer, name='Gtanh', nonlinearity=tanh)
return dict(input_layers=input_layers, output_layers=[output_layer], cur_lod=cur_lod)
def create_network(available_actions_num):
# Creates the input variables
s1 = tensor.tensor4("States")
a = tensor.vector("Actions", dtype="int32")
q2 = tensor.vector("Next State best Q-Value")
r = tensor.vector("Rewards")
nonterminal = tensor.vector("Nonterminal", dtype="int8")
# Creates the input layer of the network.
dqn = InputLayer(shape=[None, 1, downsampled_y, downsampled_x], input_var=s1)
# Adds 3 convolutional layers, each followed by a max pooling layer.
dqn = Conv2DLayer(dqn, num_filters=32, filter_size=[8, 8],
nonlinearity=rectify, W=GlorotUniform("relu"),
b=Constant(.1))
dqn = MaxPool2DLayer(dqn, pool_size=[2, 2])
dqn = Conv2DLayer(dqn, num_filters=64, filter_size=[4, 4],
nonlinearity=rectify, W=GlorotUniform("relu"),
b=Constant(.1))
dqn = MaxPool2DLayer(dqn, pool_size=[2, 2])
dqn = Conv2DLayer(dqn, num_filters=64, filter_size=[3, 3],
nonlinearity=rectify, W=GlorotUniform("relu"),
b=Constant(.1))
dqn = MaxPool2DLayer(dqn, pool_size=[2, 2])
# Adds a single fully connected layer.
dqn = DenseLayer(dqn, num_units=512, nonlinearity=rectify, W=GlorotUniform("relu"),
b=Constant(.1))
# Adds a single fully connected layer which is the output layer.
# (no nonlinearity as it is for approximating an arbitrary real function)
dqn = DenseLayer(dqn, num_units=available_actions_num, nonlinearity=None)
# Theano stuff
q = get_output(dqn)
# Only q for the chosen actions is updated more or less according to following formula:
# target Q(s,a,t) = r + gamma * max Q(s2,_,t+1)
target_q = tensor.set_subtensor(q[tensor.arange(q.shape[0]), a], r + discount_factor * nonterminal * q2)
loss = squared_error(q, target_q).mean()
# Updates the parameters according to the computed gradient using rmsprop.
params = get_all_params(dqn, trainable=True)
updates = rmsprop(loss, params, learning_rate)
# Compiles theano functions
print "Compiling the network ..."
function_learn = theano.function([s1, q2, a, r, nonterminal], loss, updates=updates, name="learn_fn")
function_get_q_values = theano.function([s1], q, name="eval_fn")
function_get_best_action = theano.function([s1], tensor.argmax(q), name="test_fn")
print "Network compiled."
# Returns Theano objects for the net and functions.
# We wouldn't need the net anymore but it is nice to save your model.
return dqn, function_learn, function_get_q_values, function_get_best_action
def ResidualModule(input_layer,
num_filters=64,
nonlinearity=rectify,
normalize=False,
stride=(1, 1),
conv_dropout=0.0):
input_conv = Conv2DLayer(incoming=input_layer,
num_filters=num_filters,
filter_size=(3, 1),
stride=stride,
pad='same',
W=lasagne.init.GlorotUniform(),
nonlinearity=None,
b=None,
name='Residual module layer 1')
l_prev = BatchNormalizeLayer(input_conv,
normalize=normalize,
nonlinearity=nonlinearity)
l_prev = TiedDropoutLayer(l_prev, p=conv_dropout, name='Tied Dropout')
l_prev = Conv2DLayer(incoming=l_prev,
num_filters=num_filters,
filter_size=(3, 1),
stride=(1, 1),
pad='same',
W=lasagne.init.GlorotUniform(),
nonlinearity=None,
b=None,
name='Residual module layer 2')
if normalize:
# Batch normalization is done "immediately after" convolutions
l_prev = BatchNormLayer(l_prev, name='Batch norm')
# Using 1x1 convolutions for shortcut projections. NiNLayer could be used as well
# but doesn't' support strides
l_skip = Conv2DLayer(input_layer,
num_filters=num_filters,
filter_size=(1, 1),
stride=stride,
nonlinearity=None,
b=None,
name='Shortcut')
l_prev = ElemwiseSumLayer((l_prev, l_skip), name='Elementwise sum')
# Add nonlinearity after summation
l_prev = NonlinearityLayer(l_prev,
nonlinearity=nonlinearity,
name='Non-linearity')
if not normalize:
l_prev = BiasLayer(l_prev, name='Bias')
l_prev = TiedDropoutLayer(l_prev, p=conv_dropout, name='Tied Dropout')
return l_prev
def UNet_decoder_2(LR_conv1, LR_conv2, LR_conv3, LR_conv4, warp_conv1, warp_conv2, warp_conv3, warp_conv4, activation = SELU_activation, W_init = W_init_SELU):
# 80
warp_deconv4 = Deconv2DLayer(ConcatLayer([LR_conv4, warp_conv4]), num_filters=64, filter_size=4, stride=2, crop=1, W = W_init, b=Constant(0.), nonlinearity=activation)
# 160
warp_deconv3 = Deconv2DLayer(ConcatLayer([warp_deconv4, LR_conv3, warp_conv3]), num_filters=64, filter_size=4, stride=2, crop=1, W = W_init, b=Constant(0.), nonlinearity=activation)
# 320
warp_deconv2 = Deconv2DLayer(ConcatLayer([warp_deconv3, LR_conv2, warp_conv2]), num_filters=64, filter_size=4, stride=2, crop=1, W = W_init, b=Constant(0.), nonlinearity=activation)
# final
post_fusion1 = Conv2DLayer(ConcatLayer([warp_deconv2, LR_conv1, warp_conv1]), 64, 5, pad=2, W = W_init, b=Constant(0.), nonlinearity=activation)
post_fusion2 = Conv2DLayer(post_fusion1, 64, 5, pad=2, W = W_init, b=Constant(0.), nonlinearity=activation)
final = Conv2DLayer(post_fusion1, 3, 5, pad=2, W = W_init_linear, b=Constant(0.), nonlinearity=linear)
return final
