def build_cnn(input_var=None, W_init=None):
"""
Builds a VGG style CNN network followed by a fully-connected layer and a softmax layer.
input_var: Theano variable for input to the network
outputs: pointer to the output of the last layer of network (softmax)
:param input_var: theano variable as input to the network
:param W_init: Initial weight values
:return: a pointer to the output of last layer
"""
weights = [] # Keeps the weights for all layers
count = 0
# If no initial weight is given, initialize with GlorotUniform
if W_init is None:
W_init = [lasagne.init.GlorotUniform()] * 7
# Input layer
network = InputLayer(shape=(None, 3, imSize, imSize),
input_var=input_var)
# CNN Stack 1
network = Conv2DLayer(network, num_filters=32, filter_size=(3, 3),
W=W_init[count], pad='same')
count += 1
weights.append(network.W)
network = Conv2DLayer(network, num_filters=32, filter_size=(3, 3),
W=W_init[count], pad='same')
count += 1
weights.append(network.W)
network = Conv2DLayer(network, num_filters=32, filter_size=(3, 3),
W=W_init[count], pad='same')
count += 1
weights.append(network.W)
network = Conv2DLayer(network, num_filters=32, filter_size=(3, 3),
W=W_init[count], pad='same')
count += 1
weights.append(network.W)
network = MaxPool2DLayer(network, pool_size=(2, 2))
# CNN Stack 2
network = Conv2DLayer(network, num_filters=64, filter_size=(3, 3),
W=W_init[count], pad='same')
count += 1
weights.append(network.W)
network = Conv2DLayer(network, num_filters=64, filter_size=(3, 3),
W=W_init[count], pad='same')
count += 1
weights.append(network.W)
network = MaxPool2DLayer(network, pool_size=(2, 2))
# CNN Stack 3
network = Conv2DLayer(network, num_filters=128, filter_size=(3, 3),
W=W_init[count], pad='same')
count += 1
weights.append(network.W)
network = MaxPool2DLayer(network, pool_size=(2, 2))
return network, weights
def SonoNet32(input_var, image_size, num_labels):
net = {}
net['input'] = InputLayer(shape=(None, 1, image_size[0], image_size[1]),
input_var=input_var)
net['conv1_1'] = batch_norm(
Conv2DLayer(net['input'], 32, 3, pad=1, flip_filters=False))
net['conv1_2'] = batch_norm(
Conv2DLayer(net['conv1_1'], 32, 3, pad=1, flip_filters=False))
net['pool1'] = MaxPool2DLayer(net['conv1_2'], 2)
net['conv2_1'] = batch_norm(
Conv2DLayer(net['pool1'], 64, 3, pad=1, flip_filters=False))
net['conv2_2'] = batch_norm(
Conv2DLayer(net['conv2_1'], 64, 3, pad=1, flip_filters=False))
net['pool2'] = MaxPool2DLayer(net['conv2_2'], 2)
net['conv3_1'] = batch_norm(
Conv2DLayer(net['pool2'], 128, 3, pad=1, flip_filters=False))
net['conv3_2'] = batch_norm(
Conv2DLayer(net['conv3_1'], 128, 3, pad=1, flip_filters=False))
net['conv3_3'] = batch_norm(
Conv2DLayer(net['conv3_2'], 128, 3, pad=1, flip_filters=False))
net['pool3'] = MaxPool2DLayer(net['conv3_3'], 2)
net['conv4_1'] = batch_norm(
Conv2DLayer(net['pool3'], 256, 3, pad=1, flip_filters=False))
net['conv4_2'] = batch_norm(
Conv2DLayer(net['conv4_1'], 256, 3, pad=1, flip_filters=False))
net['conv4_3'] = batch_norm(
Conv2DLayer(net['conv4_2'], 256, 3, pad=1, flip_filters=False))
net['pool4'] = MaxPool2DLayer(net['conv4_3'], 2)
net['conv5_1'] = batch_norm(
Conv2DLayer(net['pool4'], 256, 3, pad=1, flip_filters=False))
net['conv5_2'] = batch_norm(
Conv2DLayer(net['conv5_1'], 256, 3, pad=1, flip_filters=False))
net['conv5_3'] = batch_norm(
Conv2DLayer(net['conv5_2'], 256, 3, pad=1, flip_filters=False))
net['conv5_p'] = batch_norm(
Conv2DLayer(net['conv5_3'], num_filters=128, filter_size=(1, 1)))
net['conv6_p'] = batch_norm(
Conv2DLayer(net['conv5_p'],
num_filters=num_labels,
filter_size=(1, 1),
nonlinearity=linear))
net['average_pool_p'] = GlobalPoolLayer(net['conv6_p'],
pool_function=T.mean)
net['softmax_p'] = NonlinearityLayer(net['average_pool_p'],
nonlinearity=softmax)
net['output'] = net['softmax_p']
net['feature_maps'] = net['conv6_p']
net['last_activation'] = net['average_pool_p']
return net
def simple_network4(input_size, output_size):
l_in = InputLayer(shape=((None, ) + input_size), name='inputLayer')
#print lasagne.layers.get_output_shape(l_in)
network = lasagne.layers.Conv2DLayer(
l_in,
num_filters=32,
filter_size=(8, 8),
stride=4,
pad=2,
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.HeNormal(gain='relu'))
network = MaxPool2DLayer(network, 2)
network = lasagne.layers.Conv2DLayer(
network,
num_filters=64,
filter_size=(4, 4),
stride=2,
pad=1,
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.HeNormal(gain='relu'))
network = MaxPool2DLayer(network, 2, pad=1)
network = lasagne.layers.Conv2DLayer(
network,
num_filters=64,
filter_size=(3, 3),
stride=1,
pad=1,
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.HeNormal(gain='relu'))
network = MaxPool2DLayer(network, 2, pad=1)
network = lasagne.layers.DenseLayer(
network,
num_units=256,
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.HeNormal(gain='relu'))
network = lasagne.layers.DenseLayer(
network,
num_units=256,
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.HeNormal(gain='relu'))
network = lasagne.layers.DenseLayer(network,
num_units=output_size,
nonlinearity=None,
b=lasagne.init.Constant(.1))
return network
def build_network(input_var, num_input_channels, num_classes):
conv_defs = {
'W': lasagne.init.HeNormal('relu'),
'b': lasagne.init.Constant(0.0),
'filter_size': (3, 3),
'stride': (1, 1),
'nonlinearity': lasagne.nonlinearities.LeakyRectify(0.1)
}
nin_defs = {
'W': lasagne.init.HeNormal('relu'),
'b': lasagne.init.Constant(0.0),
'nonlinearity': lasagne.nonlinearities.LeakyRectify(0.1)
}
dense_defs = {
'W': lasagne.init.HeNormal(1.0),
'b': lasagne.init.Constant(0.0),
'nonlinearity': lasagne.nonlinearities.softmax
}
wn_defs = {
'momentum': config.batch_normalization_momentum
}
net = InputLayer ( name='input', shape=(None, num_input_channels, 28, 28), input_var=input_var)
net = GaussianNoiseLayer(net, name='noise', sigma=config.augment_noise_stddev)
net = WN(Conv2DLayer (net, name='conv1a', num_filters=32, pad='same', **conv_defs), **wn_defs)
net = WN(Conv2DLayer (net, name='conv1b', num_filters=64, pad='same', **conv_defs), **wn_defs)
# net = WN(Conv2DLayer (net, name='conv1c', num_filters=128, pad='same', **conv_defs), **wn_defs)
net = MaxPool2DLayer (net, name='pool1', pool_size=(2, 2))
net = DropoutLayer (net, name='drop1', p=.5)
net = WN(Conv2DLayer (net, name='conv2a', num_filters=32, pad='same', **conv_defs), **wn_defs)
net = WN(Conv2DLayer (net, name='conv2b', num_filters=64, pad='same', **conv_defs), **wn_defs)
# net = WN(Conv2DLayer (net, name='conv2c', num_filters=256, pad='same', **conv_defs), **wn_defs)
net = MaxPool2DLayer (net, name='pool2', pool_size=(2, 2))
net = DropoutLayer (net, name='drop2', p=.5)
net = WN(Conv2DLayer (net, name='conv3a', num_filters=32, pad=0, **conv_defs), **wn_defs)
# net = WN(NINLayer (net, name='conv3b', num_units=256, **nin_defs), **wn_defs)
net = WN(NINLayer (net, name='conv3c', num_units=256, **nin_defs), **wn_defs)
net = GlobalPoolLayer (net, name='pool3')
net = WN(DenseLayer (net, name='dense', num_units=num_classes, **dense_defs), **wn_defs)
# net = GaussianNoiseLayer(net, name='noise', sigma=config.augment_noise_stddev)
# net = WN(DenseLayer (net, name='dense1', num_units=256, **dense_defs), **wn_defs)
# net = DropoutLayer (net, name='drop1', p=.5)
# net = WN(DenseLayer (net, name='dense2', num_units=256, **dense_defs), **wn_defs)
# net = DropoutLayer (net, name='drop2', p=.5)
# net = WN(DenseLayer (net, name='dense3', num_units=256, **dense_defs), **wn_defs)
#
# net = WN(DenseLayer (net, name='dense4', num_units=num_classes, **dense_defs), **wn_defs)
return net
def create_network(available_actions_num):
# Creates the input variables
s1 = tensor.tensor4("States")
a = tensor.vector("Actions", dtype="int32")
q2 = tensor.vector("Next State best Q-Value")
r = tensor.vector("Rewards")
nonterminal = tensor.vector("Nonterminal", dtype="int8")
# Creates the input layer of the network.
dqn = InputLayer(shape=[None, 1, downsampled_y, downsampled_x], input_var=s1)
# Adds 3 convolutional layers, each followed by a max pooling layer.
dqn = Conv2DLayer(dqn, num_filters=32, filter_size=[8, 8],
nonlinearity=rectify, W=GlorotUniform("relu"),
b=Constant(.1))
dqn = MaxPool2DLayer(dqn, pool_size=[2, 2])
dqn = Conv2DLayer(dqn, num_filters=64, filter_size=[4, 4],
nonlinearity=rectify, W=GlorotUniform("relu"),
b=Constant(.1))
dqn = MaxPool2DLayer(dqn, pool_size=[2, 2])
dqn = Conv2DLayer(dqn, num_filters=64, filter_size=[3, 3],
nonlinearity=rectify, W=GlorotUniform("relu"),
b=Constant(.1))
dqn = MaxPool2DLayer(dqn, pool_size=[2, 2])
# Adds a single fully connected layer.
dqn = DenseLayer(dqn, num_units=512, nonlinearity=rectify, W=GlorotUniform("relu"),
b=Constant(.1))
# Adds a single fully connected layer which is the output layer.
# (no nonlinearity as it is for approximating an arbitrary real function)
dqn = DenseLayer(dqn, num_units=available_actions_num, nonlinearity=None)
# Theano stuff
q = get_output(dqn)
# Only q for the chosen actions is updated more or less according to following formula:
# target Q(s,a,t) = r + gamma * max Q(s2,_,t+1)
target_q = tensor.set_subtensor(q[tensor.arange(q.shape[0]), a], r + discount_factor * nonterminal * q2)
loss = squared_error(q, target_q).mean()
# Updates the parameters according to the computed gradient using rmsprop.
params = get_all_params(dqn, trainable=True)
updates = rmsprop(loss, params, learning_rate)
# Compiles theano functions
print "Compiling the network ..."
function_learn = theano.function([s1, q2, a, r, nonterminal], loss, updates=updates, name="learn_fn")
function_get_q_values = theano.function([s1], q, name="eval_fn")
function_get_best_action = theano.function([s1], tensor.argmax(q), name="test_fn")
print "Network compiled."
# Returns Theano objects for the net and functions.
# We wouldn't need the net anymore but it is nice to save your model.
return dqn, function_learn, function_get_q_values, function_get_best_action
def build_architecture(input_shape, trained_weights=None):
""" Build the Theano symbolic graph representing the CNN model.
:param input_shape: A tuple representing the input shape (h,w)
:param trained_weights: Pre-trained weights. If None, the network is initialized at random.
:return: A dictionary containing all layers
"""
net = {}
net['input'] = InputLayer(input_shape)
net['conv1'] = batch_norm(
Conv2DLayer(net['input'],
num_filters=96,
filter_size=11,
stride=4,
flip_filters=False))
net['pool1'] = MaxPool2DLayer(net['conv1'], pool_size=3, stride=2)
net['conv2'] = batch_norm(
Conv2DLayer(net['pool1'],
num_filters=256,
filter_size=5,
pad=2,
flip_filters=False))
net['pool2'] = MaxPool2DLayer(net['conv2'], pool_size=3, stride=2)
net['conv3'] = batch_norm(
Conv2DLayer(net['pool2'],
num_filters=384,
filter_size=3,
pad=1,
flip_filters=False))
net['conv4'] = batch_norm(
Conv2DLayer(net['conv3'],
num_filters=384,
filter_size=3,
pad=1,
flip_filters=False))
net['conv5'] = batch_norm(
Conv2DLayer(net['conv4'],
num_filters=256,
filter_size=3,
pad=1,
flip_filters=False))
net['pool5'] = MaxPool2DLayer(net['conv5'], pool_size=3, stride=2)
net['fc1'] = batch_norm(DenseLayer(net['pool5'], num_units=2048))
net['fc2'] = batch_norm(DenseLayer(net['fc1'], num_units=2048))
if trained_weights:
lasagne.layers.set_all_param_values(net['fc2'], trained_weights)
return net
def build_architecture(input_shape, trained_weights=None):
net = {}
net['input'] = InputLayer((None, 1, None, None))
net['large_conv1'] = batch_norm(
Conv2DLayer(net['input'],
num_filters=32,
filter_size=11,
stride=3,
pad=5,
flip_filters=False))
net['large_pool1'] = MaxPool2DLayer(net['large_conv1'],
pool_size=3,
stride=2)
net['large_conv2'] = batch_norm(
Conv2DLayer(net['large_pool1'],
num_filters=64,
filter_size=5,
pad=2,
flip_filters=False))
net['large_pool2'] = MaxPool2DLayer(net['large_conv2'], pool_size=3)
net['large_conv3'] = batch_norm(
Conv2DLayer(net['large_pool2'],
num_filters=128,
filter_size=3,
pad=1,
flip_filters=False))
net['large_conv4'] = batch_norm(
Conv2DLayer(net['large_conv3'],
num_filters=128,
filter_size=3,
pad=1,
flip_filters=False))
net['large_pool4'] = MaxPool2DLayer(net['large_conv4'], pool_size=2)
net['large_conv5'] = batch_norm(
Conv2DLayer(net['large_pool4'],
num_filters=128,
filter_size=3,
pad=1,
flip_filters=False))
net['large_pool5'] = SpatialPyramidPoolingLayer(net['large_conv5'],
implementation='kaiming')
net['fc1'] = batch_norm(DenseLayer(net['large_pool5'], num_units=2048))
net['fc2'] = batch_norm(DenseLayer(net['fc1'], num_units=2048))
if trained_weights:
lasagne.layers.set_all_param_values(net['fc2'], trained_weights)
return net
def build_neural_network():
net = {}
net['input'] = InputLayer((None, 1, 28, 28))
net['conv1'] = Conv2DLayer(net['input'], num_filters=8, filter_size=5)
net['pool1'] = MaxPool2DLayer(net['conv1'], pool_size=2)
net['conv2'] = Conv2DLayer(net['pool1'], num_filters=16, filter_size=5)
net['pool2'] = MaxPool2DLayer(net['conv2'], pool_size=3)
net['hid1'] = DenseLayer(net['pool2'], num_units=100)
net['hid2'] = DenseLayer(net['hid1'], num_units=100)
net['hid3'] = DenseLayer(net['hid2'], num_units=100)
net['out'] = DenseLayer(net['hid3'], num_units=10, nonlinearity=softmax)
return net
def build_model(self):
'''
Build Acoustic Event Net model
:return:
'''
# A architecture 41 classes
nonlin = lasagne.nonlinearities.rectify
net = {}
# channel, time. frequency
net['input'] = InputLayer(
(None, feat_shape[0], feat_shape[1], feat_shape[2]))
# ----------- 1st layer group ---------------
net['conv1a'] = ConvLayer(net['input'],
num_filters=64,
filter_size=(3, 3),
stride=1,
nonlinearity=nonlin)
net['conv1b'] = ConvLayer(net['conv1a'],
num_filters=64,
filter_size=(3, 3),
stride=1,
nonlinearity=nonlin)
net['pool1'] = MaxPool2DLayer(net['conv1b'],
pool_size=(1, 2)) # (time, freq)
# ----------- 2nd layer group ---------------
net['conv2a'] = ConvLayer(net['pool1'],
num_filters=128,
filter_size=(3, 3),
stride=1,
nonlinearity=nonlin)
net['conv2b'] = ConvLayer(net['conv2a'],
num_filters=128,
filter_size=(3, 3),
stride=1,
nonlinearity=nonlin)
net['pool2'] = MaxPool2DLayer(net['conv2b'],
pool_size=(2, 2)) # (time, freq)
# ----------- fully connected layer group ---------------
net['fc5'] = DenseLayer(net['pool2'],
num_units=1024,
nonlinearity=nonlin)
net['fc6'] = DenseLayer(net['fc5'],
num_units=1024,
nonlinearity=nonlin)
net['prob'] = DenseLayer(net['fc6'],
num_units=41,
nonlinearity=lasagne.nonlinearities.softmax)
return net
def build_overfeat(inp):
net['inp'] = InputLayer((None, 3, 231, 231), input_var=inp)
net['conv1'] = Conv2DLayer(net['inp'], 96, 11, stride=4, pad='valid')
net['pool1'] = MaxPool2DLayer(net['conv1'], 2)
net['conv2'] = Conv2DLayer(net['pool1'], 256, 5, pad='valid')
net['pool2'] = MaxPool2DLayer(net['conv2'], 2)
net['conv3'] = Conv2DLayer(net['pool2'], 512, 3, pad='same')
net['conv4'] = Conv2DLayer(net['conv3'], 1024, 3, pad='same')
net['conv5'] = Conv2DLayer(net['conv4'], 1024, 3, pad='same')
net['pool3'] = MaxPool2DLayer(net['conv5'], 2)
net['fc6'] = DenseLayer(net['pool3'], 3072)
net['fc7'] = DenseLayer(net['fc6'], 4096)
net['fc8'] = DenseLayer(net['fc7'], 1000, nonlinearity=None)
return net
def simple_network2(input_size, output_size):
l_in = InputLayer(shape=((None, ) + input_size), name='inputLayer')
network = lasagne.layers.Conv2DLayer(
l_in,
num_filters=16,
filter_size=(3, 3),
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.HeNormal(gain='relu'))
network = lasagne.layers.Conv2DLayer(
l_in,
num_filters=16,
filter_size=(3, 3),
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.HeNormal(gain='relu'))
network = MaxPool2DLayer(network, 2)
network = lasagne.layers.Conv2DLayer(
l_in,
num_filters=16,
filter_size=(3, 3),
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.HeNormal(gain='relu'))
network = MaxPool2DLayer(network, 2)
network = lasagne.layers.Conv2DLayer(
l_in,
num_filters=16,
filter_size=(3, 3),
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.HeNormal(gain='relu'))
network = DropoutLayer(network, p=0.5)
network = lasagne.layers.DenseLayer(
network,
num_units=128,
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.HeNormal(gain='relu'))
network = lasagne.layers.DenseLayer(network,
num_units=output_size,
nonlinearity=None,
b=lasagne.init.Constant(.1))
return network
def build_architecture(input_shape, trained_weights=None):
net = {}
net['input'] = InputLayer(input_shape)
net['conv1'] = batch_norm(
Conv2DLayer(net['input'],
num_filters=96,
filter_size=11,
stride=4,
flip_filters=False))
net['pool1'] = MaxPool2DLayer(net['conv1'], pool_size=3, stride=2)
net['conv2'] = batch_norm(
Conv2DLayer(net['pool1'],
num_filters=256,
filter_size=5,
pad=2,
flip_filters=False))
net['pool2'] = MaxPool2DLayer(net['conv2'], pool_size=3, stride=2)
net['conv3'] = batch_norm(
Conv2DLayer(net['pool2'],
num_filters=384,
filter_size=3,
pad=1,
flip_filters=False))
net['conv4'] = batch_norm(
Conv2DLayer(net['conv3'],
num_filters=384,
filter_size=3,
pad=1,
flip_filters=False))
net['conv5'] = batch_norm(
Conv2DLayer(net['conv4'],
num_filters=256,
filter_size=3,
pad=1,
flip_filters=False))
net['pool5'] = MaxPool2DLayer(net['conv5'], pool_size=3, stride=2)
net['fc1'] = batch_norm(DenseLayer(net['pool5'], num_units=2048))
net['fc2'] = batch_norm(DenseLayer(net['fc1'], num_units=2048))
if trained_weights:
lasagne.layers.set_all_param_values(net['fc2'], trained_weights)
return net
def _model_definition(self, net):
"""
Builds the architecture of the network
"""
he_norm = HeNormal(gain='relu')
# Input filtering and downsampling with max pooling
net = batch_norm(
Conv2DLayer(net,
num_filters=32,
filter_size=7,
pad='same',
nonlinearity=rectify,
W=he_norm))
net = MaxPool2DLayer(net, 2)
net = batch_norm(
Conv2DLayer(net,
num_filters=64,
filter_size=3,
pad='same',
nonlinearity=rectify,
W=he_norm))
net = MaxPool2DLayer(net, 2)
net = batch_norm(
Conv2DLayer(net,
num_filters=128,
filter_size=3,
pad='same',
nonlinearity=rectify,
W=he_norm))
net = batch_norm(
Conv2DLayer(net,
num_filters=128,
filter_size=3,
pad='same',
nonlinearity=rectify,
W=he_norm))
net = batch_norm(
DenseLayer(net, num_units=1024, nonlinearity=rectify, W=he_norm))
net = batch_norm(
DenseLayer(net, num_units=1024, nonlinearity=rectify, W=he_norm))
# Pooling
#net = MaxPool1DLayer(net, 1)
return net
def build_inception_module(name, input_layer, nfilters):
"""
See: https://github.com/Lasagne/Recipes/blob/master/modelzoo/googlenet.py
TODO: the pooling size is wrong here - 3x3 doesn't really make sense
on our images - consider recomputing the pooling -> also requires
recomputing all the filter sizes, etc.
"""
# nfilters: (pool_proj, 1x1, 3x3_reduce, 3x3, 5x5_reduce, 5x5)
net = {}
net['pool'] = MaxPool2DLayer(input_layer, pool_size=3, stride=1, pad=1)
net['pool_proj'] = Conv2DLayer(net['pool'], nfilters[0], 1)
net['1x1'] = Conv2DLayer(input_layer, nfilters[1], 1)
net['3x3_reduce'] = Conv2DLayer(input_layer, nfilters[2], 1)
net['3x3'] = Conv2DLayer(net['3x3_reduce'], nfilters[3], 3, pad=1)
net['5x5_reduce'] = Conv2DLayer(input_layer, nfilters[4], 1)
net['5x5'] = Conv2DLayer(net['5x5_reduce'], nfilters[5], 5, pad=2)
net['output'] = ConcatLayer([
net['1x1'],
net['3x3'],
net['5x5'],
net['pool_proj'],
])
return {'{}/{}'.format(name, k): v for k, v in net.items()}
def build_cnn(input_var=None,
w_init=None,
n_layers=(4, 2, 1),
n_filters_first=32,
imsize=32,
n_colors=3):
weights = [] # Keeps the weights for all layers
count = 0
# If no initial weight is given, initialize with GlorotUniform
if w_init is None:
w_init = [lasagne.init.GlorotUniform()] * sum(n_layers)
# Input layer
network = InputLayer(shape=(None, n_colors, imsize, imsize),
input_var=input_var)
for i, s in enumerate(n_layers):
for l in range(s):
network = Conv2DLayer(network,
num_filters=n_filters_first * (2**i),
filter_size=(3, 3),
W=w_init[count],
pad='same')
count += 1
weights.append(network.W)
network = MaxPool2DLayer(network, pool_size=(2, 2))
return network, weights
def contraction(depth, deepest):
n_filters = filter_for_depth(depth)
incoming = net['input'] if depth == 0 else net['pool{}'.format(depth -
1)]
net['conv{}_1'.format(depth)] = Conv2DLayer(incoming,
num_filters=n_filters,
filter_size=3,
pad='valid',
W=HeNormal(gain='relu'),
nonlinearity=nonlinearity)
net['conv{}_2'.format(depth)] = Conv2DLayer(
net['conv{}_1'.format(depth)],
num_filters=n_filters,
filter_size=3,
pad='valid',
W=HeNormal(gain='relu'),
nonlinearity=nonlinearity)
if P.BATCH_NORMALIZATION:
net['conv{}_2'.format(depth)] = batch_norm(
net['conv{}_2'.format(depth)],
alpha=P.BATCH_NORMALIZATION_ALPHA)
if not deepest:
net['pool{}'.format(depth)] = MaxPool2DLayer(
net['conv{}_2'.format(depth)], pool_size=2, stride=2)
def architecture(input_var, input_shape):
layer = InputLayer(input_shape, input_var)
kwargs = dict(nonlinearity=lasagne.nonlinearities.leaky_rectify,
W=lasagne.init.Orthogonal())
layer = Conv2DLayer(layer, 64, 3, **kwargs)
layer = Conv2DLayer(layer, 32, 3, **kwargs)
layer = MaxPool2DLayer(layer, 3)
layer = Conv2DLayer(layer, 128, 3, **kwargs)
layer = Conv2DLayer(layer, 64, 3, **kwargs)
layer = MaxPool2DLayer(layer, 3)
layer = DenseLayer(dropout(layer, 0.5), 256, **kwargs)
layer = DenseLayer(dropout(layer, 0.5), 64, **kwargs)
layer = DenseLayer(dropout(layer, 0.5), 1,
nonlinearity=lasagne.nonlinearities.sigmoid,
W=lasagne.init.Orthogonal())
return layer
def build_cnn(input_var=None, w_init=None, n_layers=(4, 2, 1), n_filters_first=32, imsize=32, n_colors=3):
"""
Builds a VGG style CNN network followed by a fully-connected layer and a softmax layer.
Stacks are separated by a maxpool layer. Number of kernels in each layer is twice
the number in previous stack.
input_var: Theano variable for input to the network
outputs: pointer to the output of the last layer of network (softmax)
:param input_var: theano variable as input to the network
:param w_init: Initial weight values
:param n_layers: number of layers in each stack. An array of integers with each
value corresponding to the number of layers in each stack.
(e.g. [4, 2, 1] == 3 stacks with 4, 2, and 1 layers in each.
:param n_filters_first: number of filters in the first layer
:param imSize: Size of the image
:param n_colors: Number of color channels (depth)
:return: a pointer to the output of last layer
"""
weights = [] # Keeps the weights for all layers
count = 0
# If no initial weight is given, initialize with GlorotUniform
if w_init is None:
w_init = [lasagne.init.GlorotUniform()] * sum(n_layers)
# Input layer
network = InputLayer(shape=(None, n_colors, imsize, imsize),
input_var=input_var)
for i, s in enumerate(n_layers):
for l in range(s):
network = Conv2DLayer(network, num_filters=n_filters_first * (2 ** i), filter_size=(3, 3),
W=w_init[count], pad='same')
count += 1
weights.append(network.W)
network = MaxPool2DLayer(network, pool_size=(2, 2))
return network, weights
def deep1(input_var):
net = {}
net['data'] = InputLayer(inshape, input_var=input_var)
net['conv1'] = Conv2DLayer(net['data'],
num_filters=20,
filter_size=(1, 255),
stride=(1, 16),
pad='same',
nonlinearity=rectify)
net['conv2'] = Conv2DLayer(net['conv1'],
num_filters=20,
filter_size=(1, 255),
stride=(1, 16),
pad='same',
nonlinearity=rectify)
net['pool1'] = MaxPool2DLayer(net['conv2'], pool_size=(1, 2))
net['conv3'] = Conv2DLayer(
net['pool1'],
num_filters=30,
filter_size=(1, 127),
stride=(1, 16),
pad='same',
nonlinearity=rectify) # should have dims of (23 x 32 x ?)
net['conv4'] = Conv2DLayer(net['conv3'],
num_filters=30,
filter_size=(1, 7),
stride=1,
pad='same',
nonlinearity=rectify) # dims (23 x 32 x ?)
#net['pool2'] = MaxPool2DLayer(net['conv4'], pool_size=(1,2))
net['fcl'] = DenseLayer(net['conv4'], num_units=100, nonlinearity=rectify)
net['out'] = DenseLayer(net['fcl'],
num_units=outshape,
nonlinearity=sigmoid)
return net
def build_DCNN_maxpool_softmax(input_var=None):
from lasagne.layers import Conv2DLayer, MaxPool2DLayer
print('Training the maxpool network!!')
# Define the input variable which is 4 frames of IPW fields and 4 frames of
# reflectivity fields
l_in = lasagne.layers.InputLayer(shape=(None, 8, 33, 33),
input_var=input_var)
l_conv1 = Conv2DLayer(l_in,
num_filters=32,
filter_size=(5, 5),
stride=(1, 1),
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.HeUniform(),
b=lasagne.init.Constant(.1),
pad='full')
l_maxpool = MaxPool2DLayer(l_conv1, (2, 2))
l_hidden1 = lasagne.layers.DenseLayer(
lasagne.layers.dropout(l_maxpool, p=0.3),
num_units=2000,
nonlinearity=lasagne.nonlinearities.sigmoid,
W=lasagne.init.HeUniform(),
b=lasagne.init.Constant(.1))
network = lasagne.layers.DenseLayer(
l_hidden1, num_units=2, nonlinearity=lasagne.nonlinearities.softmax)
return network, l_hidden1
def net_color_non_square(NeuralNet):
l = InputLayer(shape=(None, 3, 20, 28))
l = Conv2DLayer(l, name='conv1', filter_size=(5, 5), num_filters=8)
l = MaxPool2DLayer(l, name='pool1', pool_size=(2, 2))
l = Conv2DLayer(l, name='conv2', filter_size=(5, 5), num_filters=8)
l = MaxPool2DLayer(l, name='pool2', pool_size=(2, 2))
l = DenseLayer(l, name='hidden1', num_units=128)
l = DenseLayer(l, name='output', nonlinearity=softmax, num_units=10)
net = NeuralNet(
layers=l,
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=1,
)
return net
def build_resfuse_net(input_var=None, projection=True):
"""
Args:
projection: If False, we are using pure ResNet
Returns:
"""
# Building the network
l_in = InputLayer(shape=(None, 3, 64, 64), input_var=input_var)
# first layer, output is 64 x 64 x 64
l = batch_norm(
ConvLayer(l_in,
num_filters=64,
filter_size=(3, 3),
stride=(1, 1),
nonlinearity=rectify,
pad='same',
W=lasagne.init.HeNormal(gain='relu')))
l = MaxPool2DLayer(l, 2) # 64 x 32 x 32
# first stack of residual blocks, output is 64 x 32 x 32 (2 residual blocks) (4 conv layers)
l = resfuse_block(l, projection=projection)
l = residual_block(
l, increase_dim=True) # 128 x 16 x 16 (1 res block) (2 conv layers)
l = resfuse_block(l, projection=projection
) # 128 x 16 x 16 (2 residual blocks) (4 conv layers)
l = residual_block(
l,
increase_dim=True) # 256 x 8 x 8 (1 residual blocks) (2 conv layers)
l = resfuse_block(l, projection=projection
) # 256 x 8 x 8 (2 residual blocks) (4 conv layers)
# those are 25-layer addition (before is 19-layer config)
l = resfuse_block(l, projection=projection
) # 256 x 8 x 8 (2 residual blocks) (4 conv layers)
l = residual_block(
l, increase_dim=True) # 512 x 4 x 4 (1 res block) (2 conv layers)
# (4 + 2) * 2 + 4 * 2 + 2 + 3 = 25 layers
# average pooling
l = GlobalPoolLayer(l)
# fully connected layer
network = DenseLayer(l,
num_units=100,
W=lasagne.init.HeNormal(),
nonlinearity=softmax)
return network
def load_mnist_classifier(pkl_path):
nl = lasagne.nonlinearities.LeakyRectify(0.1)
net = InputLayer((None, 1, 32, 32))
net = Conv2DLayer(net, 32, (3, 3), pad='same', nonlinearity=nl)
net = Conv2DLayer(net, 32, (3, 3), pad='same', nonlinearity=nl)
net = MaxPool2DLayer(net, (2, 2))
net = Conv2DLayer(net, 55, (3, 3), pad='same', nonlinearity=nl)
net = Conv2DLayer(net, 55, (3, 3), pad='same', nonlinearity=nl)
net = MaxPool2DLayer(net, (2, 2))
net = Conv2DLayer(net, 96, (3, 3), pad=0, nonlinearity=nl)
net = Conv2DLayer(net, 96, (3, 3), pad=0, nonlinearity=nl)
net = MaxPool2DLayer(net, (2, 2))
net = DenseLayer(net, num_units=10, nonlinearity=lasagne.nonlinearities.softmax)
with open(pkl_path, 'rb') as file:
lasagne.layers.set_all_param_values(net, cPickle.load(file))
return net
def net(NeuralNet):
l = InputLayer(shape=(None, 1, 28, 28))
l = Conv2DLayer(l, name='conv1', filter_size=(5, 5), num_filters=8)
l = MaxPool2DLayer(l, name='pool1', pool_size=(2, 2))
l = Conv2DLayer(l, name='conv2', filter_size=(5, 5), num_filters=8)
l = MaxPool2DLayer(l, name='pool2', pool_size=(2, 2))
l = DenseLayer(l, name='hidden1', num_units=128)
l = DenseLayer(l, name='output', nonlinearity=softmax, num_units=10)
return NeuralNet(
layers=l,
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=5,
on_epoch_finished=[_OnEpochFinished()],
verbose=99,
)
def build_network():
conv_defs = {
'W': lasagne.init.HeNormal('relu'),
'b': lasagne.init.Constant(0.0),
'filter_size': (3, 3),
'stride': (1, 1),
'nonlinearity': lasagne.nonlinearities.LeakyRectify(0.1)
}
nin_defs = {
'W': lasagne.init.HeNormal('relu'),
'b': lasagne.init.Constant(0.0),
'nonlinearity': lasagne.nonlinearities.LeakyRectify(0.1)
}
dense_defs = {
'W': lasagne.init.HeNormal(1.0),
'b': lasagne.init.Constant(0.0),
'nonlinearity': lasagne.nonlinearities.softmax
}
wn_defs = {
'momentum': .999
}
net = InputLayer ( name='input', shape=(None, 3, 32, 32))
net = GaussianNoiseLayer(net, name='noise', sigma=.15)
net = WN(Conv2DLayer (net, name='conv1a', num_filters=128, pad='same', **conv_defs), **wn_defs)
net = WN(Conv2DLayer (net, name='conv1b', num_filters=128, pad='same', **conv_defs), **wn_defs)
net = WN(Conv2DLayer (net, name='conv1c', num_filters=128, pad='same', **conv_defs), **wn_defs)
net = MaxPool2DLayer (net, name='pool1', pool_size=(2, 2))
net = DropoutLayer (net, name='drop1', p=.5)
net = WN(Conv2DLayer (net, name='conv2a', num_filters=256, pad='same', **conv_defs), **wn_defs)
net = WN(Conv2DLayer (net, name='conv2b', num_filters=256, pad='same', **conv_defs), **wn_defs)
net = WN(Conv2DLayer (net, name='conv2c', num_filters=256, pad='same', **conv_defs), **wn_defs)
net = MaxPool2DLayer (net, name='pool2', pool_size=(2, 2))
net = DropoutLayer (net, name='drop2', p=.5)
net = WN(Conv2DLayer (net, name='conv3a', num_filters=512, pad=0, **conv_defs), **wn_defs)
net = WN(NINLayer (net, name='conv3b', num_units=256, **nin_defs), **wn_defs)
net = WN(NINLayer (net, name='conv3c', num_units=128, **nin_defs), **wn_defs)
net = GlobalPoolLayer (net, name='pool3')
net = WN(DenseLayer (net, name='dense', num_units=10, **dense_defs), **wn_defs)
return net
def create_model(incoming, options):
input_p = 0.2
hidden_p = 0.5
conv_num_filters1 = int(100 / (1.0 - input_p))
conv_num_filters2 = int(150 / (1.0 - hidden_p))
conv_num_filters3 = int(200 / (1.0 - hidden_p))
filter_size1 = 5
filter_size2 = 5
filter_size3 = 3
pool_size = 2
encode_size = int(options['BOTTLENECK'] / 0.5)
dense_mid_size = int(options['DENSE'] / 0.5)
pad_in = 'valid'
pad_out = 'full'
scaled_tanh = create_scaled_tanh()
dropout0 = DropoutLayer(incoming, p=0.2, name='dropout0')
conv2d1 = Conv2DLayer(dropout0, num_filters=conv_num_filters1, filter_size=filter_size1, pad=pad_in, name='conv2d1', nonlinearity=scaled_tanh)
maxpool2d2 = MaxPool2DLayer(conv2d1, pool_size=pool_size, name='maxpool2d2')
dropout1 = DropoutLayer(maxpool2d2, name='dropout1')
conv2d3 = Conv2DLayer(dropout1, num_filters=conv_num_filters2, filter_size=filter_size2, pad=pad_in, name='conv2d3', nonlinearity=scaled_tanh)
maxpool2d4 = MaxPool2DLayer(conv2d3, pool_size=pool_size, name='maxpool2d4', pad=(1,0))
dropout2 = DropoutLayer(maxpool2d4, name='dropout2')
conv2d5 = Conv2DLayer(dropout2, num_filters=conv_num_filters3, filter_size=filter_size3, pad=pad_in, name='conv2d5', nonlinearity=scaled_tanh)
reshape6 = ReshapeLayer(conv2d5, shape=([0], -1), name='reshape6') # 3000
reshape6_output = reshape6.output_shape[1]
dropout3 = DropoutLayer(reshape6, name='dropout3')
dense7 = DenseLayer(dropout3, num_units=dense_mid_size, name='dense7', nonlinearity=scaled_tanh)
dropout4 = DropoutLayer(dense7, name='dropout4')
bottleneck = DenseLayer(dropout4, num_units=encode_size, name='bottleneck', nonlinearity=linear)
# print_network(bottleneck)
dense8 = DenseLayer(bottleneck, num_units=dense_mid_size, W=bottleneck.W.T, name='dense8', nonlinearity=linear)
dense9 = DenseLayer(dense8, num_units=reshape6_output, W=dense7.W.T, nonlinearity=scaled_tanh, name='dense9')
reshape10 = ReshapeLayer(dense9, shape=([0], conv_num_filters3, 3, 5), name='reshape10') # 32 x 4 x 7
deconv2d11 = Deconv2DLayer(reshape10, conv2d5.input_shape[1], conv2d5.filter_size, stride=conv2d5.stride,
W=conv2d5.W, flip_filters=not conv2d5.flip_filters, name='deconv2d11', nonlinearity=scaled_tanh)
upscale2d12 = Upscale2DLayer(deconv2d11, scale_factor=pool_size, name='upscale2d12')
deconv2d13 = Deconv2DLayer(upscale2d12, conv2d3.input_shape[1], conv2d3.filter_size, stride=conv2d3.stride,
W=conv2d3.W, flip_filters=not conv2d3.flip_filters, name='deconv2d13', nonlinearity=scaled_tanh)
upscale2d14 = Upscale2DLayer(deconv2d13, scale_factor=pool_size, name='upscale2d14')
deconv2d15 = Deconv2DLayer(upscale2d14, conv2d1.input_shape[1], conv2d1.filter_size, stride=conv2d1.stride,
crop=(1, 0), W=conv2d1.W, flip_filters=not conv2d1.flip_filters, name='deconv2d14', nonlinearity=scaled_tanh)
reshape16 = ReshapeLayer(deconv2d15, ([0], -1), name='reshape16')
return reshape16, bottleneck
def build_model(batch_size=128):
x = T.tensor4('input')
layer = InputLayer((batch_size, 3, image_sz, image_sz), input_var=x)
layer = Conv2DLayer(layer, 64, 11, stride=4, pad='valid')
layer = MaxPool2DLayer(layer, 3, stride=2)
layer = Conv2DLayer(layer, 192, 5, pad='same')
layer = MaxPool2DLayer(layer, 3, stride=2)
layer = Conv2DLayer(layer, 384, 3, pad='same')
layer = Conv2DLayer(layer, 256, 3, pad='same')
layer = Conv2DLayer(layer, 256, 3, pad='same')
layer = MaxPool2DLayer(layer, 3, stride=2)
layer = DenseLayer(layer, 4096)
layer = DenseLayer(layer, 4096)
layer = DenseLayer(layer, 1000, nonlinearity=None)
return layer, x
def create_model(incoming, options):
conv_num_filters1 = 100
conv_num_filters2 = 150
conv_num_filters3 = 200
filter_size1 = 5
filter_size2 = 5
filter_size3 = 3
pool_size = 2
encode_size = options['BOTTLENECK']
dense_mid_size = options['DENSE']
pad_in = 'valid'
pad_out = 'full'
scaled_tanh = create_scaled_tanh()
conv2d1 = Conv2DLayer(incoming, num_filters=conv_num_filters1, filter_size=filter_size1, pad=pad_in, name='conv2d1', nonlinearity=scaled_tanh)
maxpool2d3 = MaxPool2DLayer(conv2d1, pool_size=pool_size, name='maxpool2d3')
bn2 = BatchNormLayer(maxpool2d3, name='batchnorm2')
conv2d4 = Conv2DLayer(bn2, num_filters=conv_num_filters2, filter_size=filter_size2, pad=pad_in, name='conv2d4', nonlinearity=scaled_tanh)
maxpool2d6 = MaxPool2DLayer(conv2d4, pool_size=pool_size, name='maxpool2d6', pad=(1,0))
bn3 = BatchNormLayer(maxpool2d6, name='batchnorm3')
conv2d7 = Conv2DLayer(bn3, num_filters=conv_num_filters3, filter_size=filter_size3, pad=pad_in, name='conv2d7', nonlinearity=scaled_tanh)
reshape9 = ReshapeLayer(conv2d7, shape=([0], -1), name='reshape9') # 3000
reshape9_output = reshape9.output_shape[1]
bn8 = BatchNormLayer(reshape9, name='batchnorm8')
dense10 = DenseLayer(bn8, num_units=dense_mid_size, name='dense10', nonlinearity=scaled_tanh)
bn11 = BatchNormLayer(dense10, name='batchnorm11')
bottleneck = DenseLayer(bn11, num_units=encode_size, name='bottleneck', nonlinearity=linear)
# print_network(bottleneck)
dense12 = DenseLayer(bottleneck, num_units=dense_mid_size, W=bottleneck.W.T, name='dense12', nonlinearity=linear)
dense13 = DenseLayer(dense12, num_units=reshape9_output, W=dense10.W.T, nonlinearity=scaled_tanh, name='dense13')
reshape14 = ReshapeLayer(dense13, shape=([0], conv_num_filters3, 3, 5), name='reshape14') # 32 x 4 x 7
deconv2d19 = Deconv2DLayer(reshape14, conv2d7.input_shape[1], conv2d7.filter_size, stride=conv2d7.stride,
W=conv2d7.W, flip_filters=not conv2d7.flip_filters, name='deconv2d19', nonlinearity=scaled_tanh)
upscale2d16 = Upscale2DLayer(deconv2d19, scale_factor=pool_size, name='upscale2d16')
deconv2d17 = Deconv2DLayer(upscale2d16, conv2d4.input_shape[1], conv2d4.filter_size, stride=conv2d4.stride,
W=conv2d4.W, flip_filters=not conv2d4.flip_filters, name='deconv2d17', nonlinearity=scaled_tanh)
upscale2d18 = Upscale2DLayer(deconv2d17, scale_factor=pool_size, name='upscale2d18')
deconv2d19 = Deconv2DLayer(upscale2d18, conv2d1.input_shape[1], conv2d1.filter_size, stride=conv2d1.stride,
crop=(1, 0), W=conv2d1.W, flip_filters=not conv2d1.flip_filters, name='deconv2d14', nonlinearity=scaled_tanh)
reshape20 = ReshapeLayer(deconv2d19, ([0], -1), name='reshape20')
return reshape20, bottleneck
def SmallNet(input_var, image_size, num_labels):
net = {}
net['input'] = InputLayer(shape=(None, 1, image_size[0], image_size[1]),
input_var=input_var)
net['conv1'] = Conv2DLayer(net['input'],
num_filters=32,
filter_size=(7, 7),
stride=(2, 2))
net['pool1'] = MaxPool2DLayer(net['conv1'], pool_size=(2, 2))
net['conv2'] = Conv2DLayer(net['pool1'],
num_filters=64,
filter_size=(5, 5),
stride=(2, 2))
net['pool2'] = MaxPool2DLayer(net['conv2'], pool_size=(2, 2))
net['conv3'] = Conv2DLayer(net['pool2'],
num_filters=128,
filter_size=(3, 3),
pad=(1, 1))
net['conv4'] = Conv2DLayer(net['conv3'],
num_filters=128,
filter_size=(3, 3),
pad=(1, 1))
net['conv5_p'] = Conv2DLayer(net['conv4'],
num_filters=64,
filter_size=(1, 1))
net['conv6_p'] = Conv2DLayer(net['conv5_p'],
num_filters=num_labels,
filter_size=(1, 1),
nonlinearity=linear)
net['average_pool_p'] = GlobalPoolLayer(net['conv6_p'],
pool_function=T.mean)
net['softmax_p'] = NonlinearityLayer(net['average_pool_p'],
nonlinearity=softmax)
net['output'] = net['softmax_p']
net['feature_maps'] = net['conv6_p']
net['last_activation'] = net['average_pool_p']
return net
def build_model(batch_size=128):
x = T.tensor4('input')
layer = InputLayer((batch_size, ) + featureDim, input_var=x)
layer = Conv2DLayer(layer, 96, 11, stride=4, pad=2)
layer = MaxPool2DLayer(layer, 3, stride=2)
layer = Conv2DLayer(layer, 256, 5, pad='same')
layer = MaxPool2DLayer(layer, 3, stride=2)
layer = Conv2DLayer(layer, 384, 3, pad='same')
layer = Conv2DLayer(layer, 384, 3, pad='same')
layer = Conv2DLayer(layer, 256, 3, pad='same')
layer = MaxPool2DLayer(layer, 3, stride=2)
layer = ReshapeLayer(layer, (-1, 256 * 6 * 6))
layer = DenseLayer(layer, 4096)
layer = DenseLayer(layer, 4096)
layer = DenseLayer(layer, labelDim, nonlinearity=None)
return layer, x