def auto_encoder( input_var, input_length, output_length ):
input_layer = InputLayer( shape = (None, 1, input_length ), \
input_var = input_var )
conv_1 = Conv1DLayer( input_layer, num_filters = 30, \
filter_size = 3, pad = 'same', \
name = 'conv_1' )
conv_2 = Conv1DLayer( input_layer, num_filters = 50, \
filter_size = 5, pad = 'same', \
name = 'conv_2' )
concats = ConcatLayer( [conv_1, conv_2] )
encoding = DenseLayer( concats, num_units = 50 )
dense_1 = DenseLayer( encoding, num_units = 100 )
dense_2 = DenseLayer( dense_1, num_units = 50 )
dense_3 = DenseLayer( dense_2, num_units = 10 )
decoding = DenseLayer( dense_3, num_units = output_length )
return encoding ,decoding
def __init__(self,
incoming,
filter_size=1,
pad="same",
nonlinearity=None,
**kwargs):
"""
Creates a GatedConv1DLayer instance
:param incoming: incoming layer
:param num_filters: int, number of conv filters
:param filter_size: int, size of conv filters
:param nonlinearity: activation function for h
"""
super(GatedConv1DLayer, self).__init__(incoming, **kwargs)
num_filters = incoming.output_shape[1]
h = Conv1DLayer(incoming,
num_filters,
filter_size,
pad=pad,
nonlinearity=nonlinearity,
**kwargs)
g = Conv1DLayer(incoming,
num_filters,
filter_size,
pad=pad,
nonlinearity=T.nnet.sigmoid,
**kwargs)
self.h = get_output(h)
self.g = get_output(g)
self.get_output_shape_for = g.get_output_shape_for
def build_lasagne_model(architecture, dim, input_tensor, shape):
last_layer = InputLayer(shape, input_var=input_tensor)
print('Model shape:')
for i, e in enumerate(architecture):
dropout, layers = e
print(' Layer {}:'.format(i))
ll = []
for l in layers:
filter_size, num_filters, nonlinearity, pad = l
cl = Conv1DLayer(last_layer,
num_filters=num_filters,
filter_size=(filter_size),
nonlinearity=nonlinearity_mapping[nonlinearity],
pad=pad)
ll.append(cl)
print(' - size: {}\tnum: {}'.format(filter_size, num_filters,
get_output_shape(cl)))
c = ConcatLayer(ll, axis=1)
last_layer = DropoutLayer(c, p=dropout)
print(' - dropout: {}'.format(dropout))
return Conv1DLayer(last_layer,
num_filters=1,
filter_size=3,
nonlinearity=sigmoid,
pad='same')
def cnn_model(M,
K=20,
T=1,
hh=.0001,
ep=5000,
d=0,
hsp=0.0001,
wsp=0,
spb=3,
bt=0,
al='rprop'):
# Facilitate reasonable convolutions core
theano.config.dnn.conv.algo_fwd = 'fft_tiling'
theano.config.dnn.conv.algo_bwd_filter = 'none'
theano.config.dnn.conv.algo_bwd_data = 'none'
# Reformat input data
M3 = reshape(M.astype(float32), (1, M.shape[0], M.shape[1]))
# Copy key variables to GPU
_M = Th.tensor3('_M')
# Input and forward transform
I = InputLayer(shape=M3.shape, input_var=_M)
# First layer is the transform to a non-negative subspace
H = Conv1DLayer(I,
filter_size=T,
num_filters=K,
pad='same',
nonlinearity=lambda x: psoftplus(x, spb),
b=None)
# Upper layer is the synthesizer
R = Conv1DLayer(H,
filter_size=T,
num_filters=M.shape[0],
pad='same',
nonlinearity=lambda x: psoftplus(x, spb),
b=None)
# Cost function
Ro = get_output(R) + eps
cost = Th.mean( _M*(Th.log( _M+eps) - Th.log( Ro)) - _M + Ro) \
+ hsp*Th.mean( get_output( H))
# Train it using Lasagne
opt = downhill.build(al, loss=cost, inputs=[_M], params=get_all_params(R))
train = downhill.Dataset(M3, batch_size=bt)
er = downhill_train(opt, train, hh, ep, None)
# Get approximation and hidden state
_r = squeeze(nget(R, _M, M3))
_h = squeeze(nget(H, _M, M3))
return _r, R.W.get_value(), er, _h
def cnn_fn(self):
l_in = InputLayer((None, self.max_length, self.vocab_size))
l_in_T = DimshuffleLayer(l_in, (0, 2, 1))
l_causal_conv = DilatedConv1DLayer(
l_in_T,
num_filters=self.nn_residual_channels,
dilation=1,
nonlinearity=None)
l_prev = l_causal_conv
skip_layers = []
for h in range(len(self.nn_dilations)):
l_filter = DilatedConv1DLayer(
l_prev,
num_filters=self.nn_dilation_channels,
dilation=self.nn_dilations[h],
nonlinearity=tanh)
l_gate = DilatedConv1DLayer(l_prev,
num_filters=self.nn_dilation_channels,
dilation=self.nn_dilations[h],
nonlinearity=sigmoid)
l_merge = ElemwiseMergeLayer([l_filter, l_gate],
merge_function=T.mul)
l_dense = Conv1DLayer(l_merge,
num_filters=self.nn_residual_channels,
filter_size=1,
nonlinearity=None)
l_residual = ElemwiseSumLayer([l_prev, l_dense])
l_skip = Conv1DLayer(l_merge,
num_filters=self.nn_residual_channels,
filter_size=1,
nonlinearity=None)
skip_layers.append(l_skip)
l_prev = l_residual
l_skip_sum = NonlinearityLayer(ElemwiseSumLayer(skip_layers),
nonlinearity=elu)
l_final = DimshuffleLayer(l_skip_sum, (0, 2, 1))
return l_final
def build_cnn(input_layer):
# Add a channel axis for convolutional nets
network = DimshuffleLayer(input_layer, (0, 'x', 1))
network = Conv1DLayer(network, num_filters=4, filter_size=5,
nonlinearity=rectify)
network = MaxPool1DLayer(network, pool_size=2)
network = Conv1DLayer(network, num_filters=4, filter_size=5,
nonlinearity=rectify)
network = MaxPool1DLayer(network, pool_size=2)
network = DropoutLayer(network, p=.5)
network = DenseLayer(network, num_units=256, nonlinearity=rectify)
return network
def nn_fn(self):
l_in_z = InputLayer((None, self.z_dim))
l_in_x = InputLayer((None, self.max_length, self.emb_dim))
l_in_z_reshape = ReshapeLayer(l_in_z, (
[0],
[1],
1,
))
l_in_z_rep = TileLayer(l_in_z_reshape, (1, 1, self.max_length))
l_x_pre_pad = SliceLayer(PadLayer(l_in_x, [(1, 0), (0, 0)],
batch_ndim=1),
indices=slice(0, -1),
axis=1)
l_x_pre_pad = DimshuffleLayer(l_x_pre_pad, (0, 2, 1))
l_x_pre_pad_drop = DropoutLayer(l_x_pre_pad,
self.nn_word_drop,
shared_axes=(1, ))
l_concat = ConcatLayer((l_in_z_rep, l_x_pre_pad_drop), axis=1)
l_in_d = Conv1DLayer(l_concat,
num_filters=self.nn_channels_external,
pad='same',
filter_size=1,
nonlinearity=None)
for d in self.nn_dilations:
l_cnn1 = Conv1DLayer(l_in_d,
filter_size=1,
num_filters=self.nn_channels_internal)
l_dcnn = DilatedConv1DLayer(l_cnn1,
filter_size=self.nn_filter_size,
num_filters=self.nn_channels_internal,
dilation=d)
l_cnn2 = Conv1DLayer(l_dcnn,
filter_size=1,
num_filters=self.nn_channels_external)
l_in_d = ElemwiseSumLayer([l_in_d, l_cnn2])
l_final = Conv1DLayer(l_in_d,
filter_size=1,
num_filters=self.emb_dim,
nonlinearity=None)
l_out = DimshuffleLayer(l_final, (0, 2, 1))
return (l_in_z, l_in_x), l_out
def build_cnn(k_width=5, input_var=None):
# Input layer, as usual:
l_in = InputLayer(shape=(None, 1, 30), input_var=input_var)
l_conv1 = Conv1DLayer(
incoming=l_in,
num_filters=16,
filter_size=k_width,
stride=1,
pad='same',
W=lasagne.init.Normal(std=0.02),
nonlinearity=lasagne.nonlinearities.very_leaky_rectify)
l_pool1 = Pool1DLayer(incoming=l_conv1, pool_size=2, stride=2)
l_drop1 = lasagne.layers.dropout(l_pool1, p=.2)
l_fc = lasagne.layers.DenseLayer(
l_drop1, num_units=512, nonlinearity=lasagne.nonlinearities.rectify)
l_drop2 = lasagne.layers.dropout(l_fc, p=.2)
l_out = lasagne.layers.DenseLayer(
l_drop2, num_units=2, nonlinearity=lasagne.nonlinearities.softmax)
return l_out
def build_convpool_conv1d(input_vars, nb_classes, imsize=32, n_colors=3, n_timewin=3):
"""
Builds the complete network with 1D-conv layer to integrate time from sequences of EEG images.
:param input_vars: list of EEG images (one image per time window)
:param nb_classes: number of classes
:param imsize: size of the input image (assumes a square input)
:param n_colors: number of color channels in the image
:param n_timewin: number of time windows in the snippet
:return: a pointer to the output of last layer
"""
convnets = []
w_init = None
# Build 7 parallel CNNs with shared weights
for i in range(n_timewin):
if i == 0:
convnet, w_init = build_cnn(input_vars[i], imsize=imsize, n_colors=n_colors)
else:
convnet, _ = build_cnn(input_vars[i], w_init=w_init, imsize=imsize, n_colors=n_colors)
convnets.append(FlattenLayer(convnet))
# at this point convnets shape is [numTimeWin][n_samples, features]
# we want the shape to be [n_samples, features, numTimeWin]
convpool = ConcatLayer(convnets)
convpool = ReshapeLayer(convpool, ([0], n_timewin, get_output_shape(convnets[0])[1]))
convpool = DimshuffleLayer(convpool, (0, 2, 1))
# input to 1D convlayer should be in (batch_size, num_input_channels, input_length)
convpool = Conv1DLayer(convpool, 64, 3)
# A fully-connected layer of 512 units with 50% dropout on its inputs:
convpool = DenseLayer(lasagne.layers.dropout(convpool, p=.5),
num_units=512, nonlinearity=lasagne.nonlinearities.rectify)
# And, finally, the output layer with 50% dropout on its inputs:
convpool = DenseLayer(lasagne.layers.dropout(convpool, p=.5),
num_units=nb_classes, nonlinearity=lasagne.nonlinearities.softmax)
return convpool
def build_convpool_max(l_in, emb_model, word_index, MAX_NB_WORDS, EMBEDDING_DIM, MAX_SEQUENCE_LENGTH):
# load pre-trained word embeddings into an Embedding layer
# note that we set trainable = False so as to keep the embeddings fixed
embedding_matrix = data_process.prepare_embedding(word_index, emb_model, MAX_NB_WORDS, EMBEDDING_DIM)
nb_words = min(MAX_NB_WORDS, len(word_index) + 1)
print (nb_words)
embedding_layer = lasagne.layers.EmbeddingLayer(l_in, input_size=nb_words, output_size=EMBEDDING_DIM,
W=embedding_matrix) # embedding_matrix
# embedding_layer.params[embedding_layer.W].remove('trainable')
# embedding_layer = DimshuffleLayer(embedding_layer, (0, 2, 1))
# output = get_output(embedding_layer, x)
convnets = [] # models to be merged
filter_window_sizes = [2, 3, 4]
num_filters = [100, 150, 200]
for filter_len, nb_filter in zip(filter_window_sizes, num_filters):
conv = Conv1DLayer(embedding_layer, nb_filter, filter_len, stride=1, pad='valid',
nonlinearity=lasagne.nonlinearities.rectify)
conv = lasagne.layers.MaxPool1DLayer(conv, pool_size=filter_len)
conv = lasagne.layers.FlattenLayer(conv)
dense = lasagne.layers.DenseLayer(conv, nb_filter, W=lasagne.init.GlorotUniform(),
nonlinearity=lasagne.nonlinearities.rectify)
convnets.append(dense)
print ("Conv done")
convpool = lasagne.layers.ConcatLayer(convnets, axis=1)
# convpool = ElemwiseMergeLayer(convnets, theano.tensor.maximum)
# A fully-connected layer of 512 units with 50% dropout on its inputs:
# convpool = lasagne.layers.dropout(convpool, p=.02)
# convpool = DenseLayer(convpool,num_units=512, nonlinearity=lasagne.nonlinearities.rectify)
return convpool
def build_convpool_mix(input_vars,
nb_classes,
grad_clip=110,
imsize=32,
n_colors=3,
n_timewin=7):
"""
Builds the complete network with LSTM and 1D-conv layers combined
:param input_vars: list of EEG images (one image per time window)
:param nb_classes: number of classes
:param grad_clip: the gradient messages are clipped to the given value during
the backward pass.
:param imsize: size of the input image (assumes a square input)
:param n_colors: number of color channels in the image
:param n_timewin: number of time windows in the snippet
:return: a pointer to the output of last layer
"""
convnets = []
w_init = None
# Build 7 parallel CNNs with shared weights
for i in range(n_timewin):
if i == 0:
convnet, w_init = build_cnn(input_vars[i],
imsize=imsize,
n_colors=n_colors)
else:
convnet, _ = build_cnn(input_vars[i],
w_init=w_init,
imsize=imsize,
n_colors=n_colors)
convnets.append(FlattenLayer(convnet))
# at this point convnets shape is [numTimeWin][n_samples, features]
# we want the shape to be [n_samples, features, numTimeWin]
convpool = ConcatLayer(convnets)
convpool = ReshapeLayer(convpool,
([0], n_timewin, get_output_shape(convnets[0])[1]))
reformConvpool = DimshuffleLayer(convpool, (0, 2, 1))
# input to 1D convlayer should be in (batch_size, num_input_channels, input_length)
conv_out = Conv1DLayer(reformConvpool, 64, 3)
conv_out = FlattenLayer(conv_out)
# Input to LSTM should have the shape as (batch size, SEQ_LENGTH, num_features)
lstm = LSTMLayer(convpool,
num_units=128,
grad_clipping=grad_clip,
nonlinearity=lasagne.nonlinearities.tanh)
lstm_out = SliceLayer(lstm, -1, 1)
# Merge 1D-Conv and LSTM outputs
dense_input = ConcatLayer([conv_out, lstm_out])
# A fully-connected layer of 256 units with 50% dropout on its inputs:
convpool = DenseLayer(lasagne.layers.dropout(dense_input, p=.5),
num_units=512,
nonlinearity=lasagne.nonlinearities.rectify)
# And, finally, the 10-unit output layer with 50% dropout on its inputs:
convpool = DenseLayer(convpool,
num_units=nb_classes,
nonlinearity=lasagne.nonlinearities.softmax)
return convpool
def build_convpool_mix(input_vars, input_shape=None):
"""
Builds the complete network with LSTM and 1D-conv layers combined
to integrate time from sequences of EEG images.
:param input_vars: list of EEG images (one image per time window)
:return: a pointer to the output of last layer
"""
convnets = []
W_init = None
# Build 7 parallel CNNs with shared weights
for i in range(input_shape[0]):
if i == 0:
convnet, W_init = build_cnn(input_vars[i], input_shape)
else:
convnet, _ = build_cnn(input_vars[i], input_shape, W_init)
convnets.append(FlattenLayer(convnet))
# at this point convnets shape is [numTimeWin][n_samples, features]
# we want the shape to be [n_samples, features, numTimeWin]
convpool = ConcatLayer(convnets)
# convpool = ReshapeLayer(convpool, ([0], -1, numTimeWin))
convpool = ReshapeLayer(
convpool, ([0], input_shape[0], get_output_shape(convnets[0])[1]))
reformConvpool = DimshuffleLayer(convpool, (0, 2, 1))
# input to 1D convlayer should be in (batch_size, num_input_channels, input_length)
conv_out = Conv1DLayer(reformConvpool, 64, 3)
conv_out = FlattenLayer(conv_out)
# Input to LSTM should have the shape as (batch size, SEQ_LENGTH, num_features)
lstm = LSTMLayer(convpool,
num_units=128,
grad_clipping=grad_clip,
nonlinearity=lasagne.nonlinearities.tanh)
# After LSTM layer you either need to reshape or slice it (depending on whether you
# want to keep all predictions or just the last prediction.
# http://lasagne.readthedocs.org/en/latest/modules/layers/recurrent.html
# https://github.com/Lasagne/Recipes/blob/master/examples/lstm_text_generation.py
lstm_out = SliceLayer(lstm, -1, 1)
# Merge 1D-Conv and LSTM outputs
dense_input = ConcatLayer([conv_out, lstm_out])
# A fully-connected layer of 256 units with 50% dropout on its inputs:
convpool = DenseLayer(lasagne.layers.dropout(dense_input, p=.5),
num_units=512,
nonlinearity=lasagne.nonlinearities.rectify)
# We only need the final prediction, we isolate that quantity and feed it
# to the next layer.
# And, finally, the 10-unit output layer with 50% dropout on its inputs:
convpool = DenseLayer(convpool,
num_units=num_classes,
nonlinearity=lasagne.nonlinearities.softmax)
return convpool
def build_convpool_max(l_in, emb_model, word_index, MAX_NB_WORDS,
EMBEDDING_DIM, MAX_SEQUENCE_LENGTH):
"""
Builds the complete network with maxpooling layer in time.
:param input_vars: list of EEG images (one image per time window)
:param nb_classes: number of classes
:return: a pointer to the output of last layer
"""
# load pre-trained word embeddings into an Embedding layer
# note that we set trainable = False so as to keep the embeddings fixed
embedding_matrix = data_process.prepare_embedding(word_index, emb_model,
MAX_NB_WORDS,
EMBEDDING_DIM)
nb_words = min(MAX_NB_WORDS, len(word_index) + 1)
print(nb_words)
embedding_layer = lasagne.layers.EmbeddingLayer(
l_in,
input_size=nb_words,
output_size=EMBEDDING_DIM,
W=embedding_matrix) # embedding_matrix
# embedding_layer.params[embedding_layer.W].remove('trainable')
# embedding_layer = DimshuffleLayer(embedding_layer, (0, 2, 1))
# output = get_output(embedding_layer, x)
convnets = [] # models to be merged
filter_window_sizes = [2, 3, 4]
num_filters = [100, 150, 200]
for filter_len, nb_filter in zip(filter_window_sizes, num_filters):
conv = Conv1DLayer(embedding_layer,
nb_filter,
filter_len,
stride=1,
pad='valid',
nonlinearity=lasagne.nonlinearities.rectify)
conv = lasagne.layers.MaxPool1DLayer(conv, pool_size=filter_len)
conv = lasagne.layers.FlattenLayer(conv)
dense = lasagne.layers.DenseLayer(
conv,
nb_filter,
W=lasagne.init.GlorotUniform(),
nonlinearity=lasagne.nonlinearities.rectify)
convnets.append(dense)
print("Conv done")
convpool = lasagne.layers.ConcatLayer(convnets, axis=1)
# convpool = lasagne.layers.dropout(convpool, p=.02)
# convpool = DenseLayer(convpool,num_units=512, nonlinearity=lasagne.nonlinearities.rectify)
return convpool
def __init__(self,
incoming,
num_filters,
filter_size,
stride=1,
pad=0,
untie_biases=False,
W=init.GlorotUniform(),
b=init.Constant(0.),
nonlinearity=nonlinearities.rectify,
flip_filters=True,
convolution=conv.conv1d_mc0,
**kwargs):
if isinstance(incoming, tuple):
input_shape = incoming
else:
input_shape = incoming.output_shape
# Retrieve the supplied name, if it exists; otherwise use ''
if 'name' in kwargs:
basename = kwargs['name'] + '.'
# Create a separate version of kwargs for the contained layers
# which does not include 'name'
layer_kwargs = dict(
(key, arg) for key, arg in kwargs.items() if key != 'name')
else:
basename = ''
layer_kwargs = kwargs
self.conv1d = Conv1DLayer(InputLayer((None, ) + input_shape[2:]),
num_filters,
filter_size,
stride,
pad,
untie_biases,
W,
b,
nonlinearity,
flip_filters,
convolution,
name=basename + "conv1d",
**layer_kwargs)
self.W = self.conv1d.W
self.b = self.conv1d.b
super(ConvTimeStep1DLayer, self).__init__(incoming, **kwargs)
def build_cnn(input_var=None):
# Input layer, as usual:
l_in = InputLayer(shape=(None, 64, 512), input_var=input_var)
l_conv1 = Conv1DLayer(
incoming=l_in,
num_filters=128,
filter_size=3,
stride=1,
pad='same',
W=lasagne.init.Normal(std=0.02),
nonlinearity=lasagne.nonlinearities.very_leaky_rectify)
l_pool1 = Pool1DLayer(incoming=l_conv1, pool_size=4, stride=4)
# A fully-connected layer
l_fc = lasagne.layers.DenseLayer(
l_pool1, num_units=512, nonlinearity=lasagne.nonlinearities.rectify)
l_out = lasagne.layers.DenseLayer(
l_fc, num_units=2, nonlinearity=lasagne.nonlinearities.softmax)
return l_out
def event_span_classifier(args, input_var, target_var, wordEmbeddings, seqlen, num_feats):
print("Building model with 1D Convolution")
vocab_size = wordEmbeddings.shape[1]
wordDim = wordEmbeddings.shape[0]
kw = 2
num_filters = seqlen-kw+1
stride = 1
#important context words as channels
#CNN_sentence config
filter_size=wordDim
pool_size=seqlen-filter_size+1
input = InputLayer((None, seqlen, num_feats),input_var=input_var)
batchsize, _, _ = input.input_var.shape
emb = EmbeddingLayer(input, input_size=vocab_size, output_size=wordDim, W=wordEmbeddings.T)
#emb.params[emb.W].remove('trainable') #(batchsize, seqlen, wordDim)
#print get_output_shape(emb)
reshape = ReshapeLayer(emb, (batchsize, seqlen, num_feats*wordDim))
#print get_output_shape(reshape)
conv1d = Conv1DLayer(reshape, num_filters=num_filters, filter_size=wordDim, stride=1,
nonlinearity=tanh,W=GlorotUniform()) #nOutputFrame = num_flters,
#nOutputFrameSize = (num_feats*wordDim-filter_size)/stride +1
#print get_output_shape(conv1d)
conv1d = DimshuffleLayer(conv1d, (0,2,1))
#print get_output_shape(conv1d)
pool_size=num_filters
maxpool = MaxPool1DLayer(conv1d, pool_size=pool_size)
#print get_output_shape(maxpool)
#forward = FlattenLayer(maxpool)
#print get_output_shape(forward)
hid = DenseLayer(maxpool, num_units=args.hiddenDim, nonlinearity=sigmoid)
network = DenseLayer(hid, num_units=2, nonlinearity=softmax)
prediction = get_output(network)
loss = T.mean(binary_crossentropy(prediction,target_var))
lambda_val = 0.5 * 1e-4
layers = {emb:lambda_val, conv1d:lambda_val, hid:lambda_val, network:lambda_val}
penalty = regularize_layer_params_weighted(layers, l2)
loss = loss + penalty
params = get_all_params(network, trainable=True)
if args.optimizer == "sgd":
updates = sgd(loss, params, learning_rate=args.step)
elif args.optimizer == "adagrad":
updates = adagrad(loss, params, learning_rate=args.step)
elif args.optimizer == "adadelta":
updates = adadelta(loss, params, learning_rate=args.step)
elif args.optimizer == "nesterov":
updates = nesterov_momentum(loss, params, learning_rate=args.step)
elif args.optimizer == "rms":
updates = rmsprop(loss, params, learning_rate=args.step)
elif args.optimizer == "adam":
updates = adam(loss, params, learning_rate=args.step)
else:
raise "Need set optimizer correctly"
test_prediction = get_output(network, deterministic=True)
test_loss = T.mean(binary_crossentropy(test_prediction,target_var))
train_fn = theano.function([input_var, target_var],
loss, updates=updates, allow_input_downcast=True)
test_acc = T.mean(binary_accuracy(test_prediction, target_var))
val_fn = theano.function([input_var, target_var], [test_loss, test_acc], allow_input_downcast=True)
return train_fn, val_fn, network
def dense_fused_convnets(self,
fusion_level,
fusion_type,
input_var1=None,
input_var2=None,
bottleneck_W=None,
weights_dir=None):
net = OrderedDict()
net['input_rgb'] = InputLayer((None, 4, 128, 128),
input_var=input_var1)
layer = 0
for i in range(self._net_specs_dict['num_conv_layers']):
# Add convolution layers
net['conv_rgb{0:d}'.format(i + 1)] = Conv2DLayer(
net.values()[layer],
num_filters=self._net_specs_dict['num_conv_filters'][i],
filter_size=(self._net_specs_dict['conv_filter_size'][i], ) *
2,
pad='same')
layer += 1
if self._net_specs_dict['num_conv_layers'] <= 2:
# Add pooling layers
net['pool_rgb{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
else:
if i < 4:
if (i + 1) % 2 == 0:
# Add pooling layers
net['pool_rgb{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
else:
if (i + 1) == 7:
# Add pooling layers
net['pool_rgb{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
# Fc-layers
net['fc1_rgb'] = DenseLayer(net.values()[layer],
self._net_specs_dict['num_fc_units'][0])
layer += 1
if fusion_level == 2:
# Add dropout layer
net['dropout1_rgb'] = dropout(net['fc1_rgb'],
p=self._model_hp_dict['p'])
layer += 1
net['fc2_rgb'] = DenseLayer(
net['dropout1_rgb'], self._net_specs_dict['num_fc_units'][1])
layer += 1
net['input_depth'] = InputLayer((None, 1, 128, 128),
input_var=input_var2)
layer += 1
for i in range(self._net_specs_dict['num_conv_layers']):
# Add convolution layers
net['conv_depth{0:d}'.format(i + 1)] = Conv2DLayer(
net.values()[layer],
num_filters=self._net_specs_dict['num_conv_filters'][i],
filter_size=(self._net_specs_dict['conv_filter_size'][i], ) *
2,
pad='same')
layer += 1
if self._net_specs_dict['num_conv_layers'] <= 2:
# Add pooling layers
net['pool_depth{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
else:
if i < 4:
if (i + 1) % 2 == 0:
# Add pooling layers
net['pool_depth{0:d}'.format(i+1)] =\
MaxPool2DLayer(net.values()[layer],
pool_size=(3, 3))
layer += 1
else:
if (i + 1) == 7:
# Add pooling layers
net['pool_depth{0:d}'.format(i+1)] =\
MaxPool2DLayer(net.values()[layer],
pool_size=(3, 3))
layer += 1
# Fc-layers
net['fc1_depth'] = DenseLayer(net.values()[layer],
self._net_specs_dict['num_fc_units'][0])
layer += 1
if fusion_level == 2:
# Add dropout layer
net['dropout1_depth'] = dropout(net['fc1_depth'],
p=self._model_hp_dict['p'])
layer += 1
net['fc2_depth'] = DenseLayer(
net['dropout1_depth'], self._net_specs_dict['num_fc_units'][1])
layer += 1
# Fuse ConvNets by fusion_level and fusion_type
if fusion_type == self.MAX:
net['merge'] =\
ElemwiseMergeLayer([net['fc%i_rgb' % fusion_level],
net['fc%i_depth' % fusion_level]],
T.maximum)
layer += 1
elif fusion_type == self.SUM:
net['merge'] =\
ElemwiseMergeLayer([net['fc%i_rgb' % fusion_level],
net['fc%i_depth' % fusion_level]],
T.add)
layer += 1
elif fusion_type == self.CONCAT:
net['merge'] = concat([
net['fc%i_rgb' % fusion_level],
net['fc%i_depth' % fusion_level]
])
layer += 1
elif fusion_type == self.CONCATCONV:
net['fc%i_rgb_res' % fusion_level] =\
reshape(net['fc%i_rgb' % fusion_level], ([0], 1, [1]))
layer += 1
net['fc%i_depth_res' % fusion_level] =\
reshape(net['fc%i_depth' % fusion_level], ([0], 1, [1]))
layer += 1
net['concat'] = concat([
net['fc%i_rgb_res' % fusion_level],
net['fc%i_depth_res' % fusion_level]
])
layer += 1
net['merge_con'] = Conv1DLayer(net['concat'],
num_filters=1,
filter_size=(1, ),
nonlinearity=None)
layer += 1
net['merge'] = reshape(net['merge_con'], ([0], [2]))
layer += 1
if fusion_level == 1:
# Add dropout layer
net['dropout1'] = dropout(net['merge'], p=self._model_hp_dict['p'])
layer += 1
net['fc2'] = DenseLayer(net['dropout1'],
self._net_specs_dict['num_fc_units'][1])
layer += 1
# Add dropout layer
net['dropout2'] = dropout(net['fc2'], p=self._model_hp_dict['p'])
layer += 1
else:
# Add dropout layer
net['dropout2'] = dropout(net['merge'], p=self._model_hp_dict['p'])
layer += 1
# Add output layer(linear activation because it's regression)
if bottleneck_W is not None:
# Add bottleneck layer
net['bottleneck'] = DenseLayer(net['dropout2'], 30)
# Add output layer(linear activation because it's regression)
net['output'] = DenseLayer(
net['bottleneck'],
3 * self._num_joints,
W=bottleneck_W[0:30],
nonlinearity=lasagne.nonlinearities.tanh)
else:
# Add output layer(linear activation because it's regression)
net['output'] = DenseLayer(
net['dropout2'],
3 * self._num_joints,
nonlinearity=lasagne.nonlinearities.tanh)
if weights_dir is not None:
lw = LoadWeights(weights_dir, net)
lw.load_weights_numpy()
return net
def multi_task_classifier(args,
input_var,
target_var,
wordEmbeddings,
seqlen,
num_feats,
lambda_val=0.5 * 1e-4):
print("Building multi task model with 1D Convolution")
vocab_size = wordEmbeddings.shape[1]
wordDim = wordEmbeddings.shape[0]
kw = 2
num_filters = seqlen - kw + 1
stride = 1
filter_size = wordDim
pool_size = num_filters
input = InputLayer((None, seqlen, num_feats), input_var=input_var)
batchsize, _, _ = input.input_var.shape
#span
emb1 = EmbeddingLayer(input,
input_size=vocab_size,
output_size=wordDim,
W=wordEmbeddings.T)
reshape1 = ReshapeLayer(emb1, (batchsize, seqlen, num_feats * wordDim))
conv1d_1 = DimshuffleLayer(
Conv1DLayer(reshape1,
num_filters=num_filters,
filter_size=wordDim,
stride=1,
nonlinearity=tanh,
W=GlorotUniform()), (0, 2, 1))
maxpool_1 = MaxPool1DLayer(conv1d_1, pool_size=pool_size)
hid_1 = DenseLayer(maxpool_1,
num_units=args.hiddenDim,
nonlinearity=sigmoid)
network_1 = DenseLayer(hid_1, num_units=2, nonlinearity=softmax)
"""
#DocTimeRel
emb2 = EmbeddingLayer(input, input_size=vocab_size, output_size=wordDim, W=wordEmbeddings.T)
reshape2 = ReshapeLayer(emb2, (batchsize, seqlen, num_feats*wordDim))
conv1d_2 = DimshuffleLayer(Conv1DLayer(reshape2, num_filters=num_filters, filter_size=wordDim, stride=1,
nonlinearity=tanh,W=GlorotUniform()), (0,2,1))
maxpool_2 = MaxPool1DLayer(conv1d_2, pool_size=pool_size)
hid_2 = DenseLayer(maxpool_2, num_units=args.hiddenDim, nonlinearity=sigmoid)
network_2 = DenseLayer(hid_2, num_units=5, nonlinearity=softmax)
"""
#Type
emb3 = EmbeddingLayer(input,
input_size=vocab_size,
output_size=wordDim,
W=wordEmbeddings.T)
reshape3 = ReshapeLayer(emb3, (batchsize, seqlen, num_feats * wordDim))
conv1d_3 = DimshuffleLayer(
Conv1DLayer(reshape3,
num_filters=num_filters,
filter_size=wordDim,
stride=1,
nonlinearity=tanh,
W=GlorotUniform()), (0, 2, 1))
maxpool_3 = MaxPool1DLayer(conv1d_3, pool_size=pool_size)
hid_3 = DenseLayer(maxpool_3,
num_units=args.hiddenDim,
nonlinearity=sigmoid)
network_3 = DenseLayer(hid_3, num_units=4, nonlinearity=softmax)
#Degree
emb4 = EmbeddingLayer(input,
input_size=vocab_size,
output_size=wordDim,
W=wordEmbeddings.T)
reshape4 = ReshapeLayer(emb4, (batchsize, seqlen, num_feats * wordDim))
conv1d_4 = DimshuffleLayer(
Conv1DLayer(reshape4,
num_filters=num_filters,
filter_size=wordDim,
stride=1,
nonlinearity=tanh,
W=GlorotUniform()), (0, 2, 1))
maxpool_4 = MaxPool1DLayer(conv1d_4, pool_size=pool_size)
hid_4 = DenseLayer(maxpool_4,
num_units=args.hiddenDim,
nonlinearity=sigmoid)
network_4 = DenseLayer(hid_4, num_units=4, nonlinearity=softmax)
#Polarity
emb5 = EmbeddingLayer(input,
input_size=vocab_size,
output_size=wordDim,
W=wordEmbeddings.T)
reshape5 = ReshapeLayer(emb5, (batchsize, seqlen, num_feats * wordDim))
conv1d_5 = DimshuffleLayer(
Conv1DLayer(reshape5,
num_filters=num_filters,
filter_size=wordDim,
stride=1,
nonlinearity=tanh,
W=GlorotUniform()), (0, 2, 1))
maxpool_5 = MaxPool1DLayer(conv1d_5, pool_size=pool_size)
hid_5 = DenseLayer(maxpool_5,
num_units=args.hiddenDim,
nonlinearity=sigmoid)
network_5 = DenseLayer(hid_5, num_units=3, nonlinearity=softmax)
#ContextualModality
emb6 = EmbeddingLayer(input,
input_size=vocab_size,
output_size=wordDim,
W=wordEmbeddings.T)
reshape6 = ReshapeLayer(emb6, (batchsize, seqlen, num_feats * wordDim))
conv1d_6 = DimshuffleLayer(
Conv1DLayer(reshape6,
num_filters=num_filters,
filter_size=wordDim,
stride=1,
nonlinearity=tanh,
W=GlorotUniform()), (0, 2, 1))
maxpool_6 = MaxPool1DLayer(conv1d_6, pool_size=pool_size)
hid_6 = DenseLayer(maxpool_6,
num_units=args.hiddenDim,
nonlinearity=sigmoid)
network_6 = DenseLayer(hid_6, num_units=5, nonlinearity=softmax)
"""
#ContextualAspect
emb7 = EmbeddingLayer(input, input_size=vocab_size, output_size=wordDim, W=wordEmbeddings.T)
reshape7 = ReshapeLayer(emb7, (batchsize, seqlen, num_feats*wordDim))
conv1d_7 = DimshuffleLayer(Conv1DLayer(reshape7, num_filters=num_filters, filter_size=wordDim, stride=1,
nonlinearity=tanh,W=GlorotUniform()), (0,2,1))
maxpool_7 = MaxPool1DLayer(conv1d_7, pool_size=pool_size)
hid_7 = DenseLayer(maxpool_7, num_units=args.hiddenDim, nonlinearity=sigmoid)
network_7 = DenseLayer(hid_7, num_units=4, nonlinearity=softmax)
"""
"""
#Permanence
emb8 = EmbeddingLayer(input, input_size=vocab_size, output_size=wordDim, W=wordEmbeddings.T)
reshape8 = ReshapeLayer(emb8, (batchsize, seqlen, num_feats*wordDim))
conv1d_8 = DimshuffleLayer(Conv1DLayer(reshape8, num_filters=num_filters, filter_size=wordDim, stride=1,
nonlinearity=tanh,W=GlorotUniform()), (0,2,1))
maxpool_8 = MaxPool1DLayer(conv1d_8, pool_size=pool_size)
hid_8 = DenseLayer(maxpool_8, num_units=args.hiddenDim, nonlinearity=sigmoid)
network_8 = DenseLayer(hid_8, num_units=4, nonlinearity=softmax)
"""
# Is this important?
"""
network_1_out, network_2_out, network_3_out, network_4_out, \
network_5_out, network_6_out, network_7_out, network_8_out = \
get_output([network_1, network_2, network_3, network_4, network_5, network_6, network_7, network_8])
"""
network_1_out = get_output(network_1)
network_3_out = get_output(network_3)
network_4_out = get_output(network_4)
network_5_out = get_output(network_5)
network_6_out = get_output(network_6)
loss_1 = T.mean(binary_crossentropy(
network_1_out, target_var)) + regularize_layer_params_weighted(
{
emb1: lambda_val,
conv1d_1: lambda_val,
hid_1: lambda_val,
network_1: lambda_val
}, l2)
updates_1 = adagrad(loss_1,
get_all_params(network_1, trainable=True),
learning_rate=args.step)
train_fn_1 = theano.function([input_var, target_var],
loss_1,
updates=updates_1,
allow_input_downcast=True)
val_acc_1 = T.mean(
binary_accuracy(get_output(network_1, deterministic=True), target_var))
val_fn_1 = theano.function([input_var, target_var],
val_acc_1,
allow_input_downcast=True)
"""
loss_2 = T.mean(categorical_crossentropy(network_2_out,target_var)) + regularize_layer_params_weighted({emb2:lambda_val, conv1d_2:lambda_val,
hid_2:lambda_val, network_2:lambda_val} , l2)
updates_2 = adagrad(loss_2, get_all_params(network_2, trainable=True), learning_rate=args.step)
train_fn_2 = theano.function([input_var, target_var], loss_2, updates=updates_2, allow_input_downcast=True)
val_acc_2 = T.mean(categorical_accuracy(get_output(network_2, deterministic=True), target_var))
val_fn_2 = theano.function([input_var, target_var], val_acc_2, allow_input_downcast=True)
"""
loss_3 = T.mean(categorical_crossentropy(
network_3_out, target_var)) + regularize_layer_params_weighted(
{
emb3: lambda_val,
conv1d_3: lambda_val,
hid_3: lambda_val,
network_3: lambda_val
}, l2)
updates_3 = adagrad(loss_3,
get_all_params(network_3, trainable=True),
learning_rate=args.step)
train_fn_3 = theano.function([input_var, target_var],
loss_3,
updates=updates_3,
allow_input_downcast=True)
val_acc_3 = T.mean(
categorical_accuracy(get_output(network_3, deterministic=True),
target_var))
val_fn_3 = theano.function([input_var, target_var],
val_acc_3,
allow_input_downcast=True)
loss_4 = T.mean(categorical_crossentropy(
network_4_out, target_var)) + regularize_layer_params_weighted(
{
emb4: lambda_val,
conv1d_4: lambda_val,
hid_4: lambda_val,
network_4: lambda_val
}, l2)
updates_4 = adagrad(loss_4,
get_all_params(network_4, trainable=True),
learning_rate=args.step)
train_fn_4 = theano.function([input_var, target_var],
loss_4,
updates=updates_4,
allow_input_downcast=True)
val_acc_4 = T.mean(
categorical_accuracy(get_output(network_4, deterministic=True),
target_var))
val_fn_4 = theano.function([input_var, target_var],
val_acc_4,
allow_input_downcast=True)
loss_5 = T.mean(categorical_crossentropy(
network_5_out, target_var)) + regularize_layer_params_weighted(
{
emb5: lambda_val,
conv1d_5: lambda_val,
hid_5: lambda_val,
network_5: lambda_val
}, l2)
updates_5 = adagrad(loss_5,
get_all_params(network_5, trainable=True),
learning_rate=args.step)
train_fn_5 = theano.function([input_var, target_var],
loss_5,
updates=updates_5,
allow_input_downcast=True)
val_acc_5 = T.mean(
categorical_accuracy(get_output(network_5, deterministic=True),
target_var))
val_fn_5 = theano.function([input_var, target_var],
val_acc_5,
allow_input_downcast=True)
loss_6 = T.mean(categorical_crossentropy(
network_6_out, target_var)) + regularize_layer_params_weighted(
{
emb6: lambda_val,
conv1d_6: lambda_val,
hid_6: lambda_val,
network_6: lambda_val
}, l2)
updates_6 = adagrad(loss_6,
get_all_params(network_6, trainable=True),
learning_rate=args.step)
train_fn_6 = theano.function([input_var, target_var],
loss_6,
updates=updates_6,
allow_input_downcast=True)
val_acc_6 = T.mean(
categorical_accuracy(get_output(network_6, deterministic=True),
target_var))
val_fn_6 = theano.function([input_var, target_var],
val_acc_6,
allow_input_downcast=True)
"""
loss_7 = T.mean(categorical_crossentropy(network_7_out,target_var)) + regularize_layer_params_weighted({emb7:lambda_val, conv1d_7:lambda_val,
hid_7:lambda_val, network_7:lambda_val} , l2)
updates_7 = adagrad(loss_7, get_all_params(network_7, trainable=True), learning_rate=args.step)
train_fn_7 = theano.function([input_var, target_var], loss_7, updates=updates_7, allow_input_downcast=True)
val_acc_7 = T.mean(categorical_accuracy(get_output(network_7, deterministic=True), target_var))
val_fn_7 = theano.function([input_var, target_var], val_acc_7, allow_input_downcast=True)
loss_8 = T.mean(categorical_crossentropy(network_8_out,target_var)) + regularize_layer_params_weighted({emb8:lambda_val, conv1d_8:lambda_val,
hid_8:lambda_val, network_8:lambda_val} , l2)
updates_8 = adagrad(loss_8, get_all_params(network_8, trainable=True), learning_rate=args.step)
train_fn_8 = theano.function([input_var, target_var], loss_8, updates=updates_8, allow_input_downcast=True)
val_acc_8 = T.mean(categorical_accuracy(get_output(network_8, deterministic=True), target_var))
val_fn_8 = theano.function([input_var, target_var], val_acc_8, allow_input_downcast=True)
"""
"""
return train_fn_1, val_fn_1, network_1, train_fn_2, val_fn_2, network_2, train_fn_3, val_fn_3, \
network_3, train_fn_4, val_fn_4, network_4, train_fn_5, val_fn_5, network_5, \
train_fn_6, val_fn_6, network_6, train_fn_7, val_fn_7, network_7, train_fn_8, val_fn_8, network_8
"""
return train_fn_1, val_fn_1, network_1, train_fn_3, val_fn_3, \
network_3, train_fn_4, val_fn_4, network_4, train_fn_5, val_fn_5, network_5, \
train_fn_6, val_fn_6, network_6
def build_MC_DCNN(input_var=[None] * parameters.N_CHANNELS):
conv_num_filters1 = 4
conv_num_filters2 = 4
filter_size1 = 5
filter_size2 = 5
pool_size = 4
pad_in = 'valid'
pad_out = 'full'
dense_units = 32
########################
# Here we build a dictionnary to construct independent layers for each channel:
network = {}
for i in range(parameters.N_CHANNELS):
network[i] = InputLayer(shape=(batch_size, 1, parameters.FREQ_WINDOW),
input_var=input_var[i],
name="input_layer_1")
network[i] = batch_norm(
Conv1DLayer(network[i],
num_filters=conv_num_filters1,
filter_size=filter_size1,
nonlinearity=lasagne.nonlinearities.tanh,
W=lasagne.init.GlorotUniform(),
pad=pad_in,
name="conv1_1"))
network[i] = MaxPool1DLayer(network[i],
pool_size=pool_size,
name="pool1_1")
network[i] = batch_norm(
Conv1DLayer(network[i],
num_filters=conv_num_filters2,
filter_size=filter_size2,
nonlinearity=lasagne.nonlinearities.tanh,
W=lasagne.init.GlorotUniform(),
pad=pad_in,
name="conv2_1"))
network[i] = MaxPool1DLayer(network[i],
pool_size=pool_size,
name="pool2_1")
network[i] = ReshapeLayer(network[i],
shape=([0], -1),
name="reshape_1")
#######################
# Now we concatenate the output from each channel layer, and build a MLP:
network2 = lasagne.layers.ConcatLayer(network.values(),
axis=1,
name="concat")
network2 = batch_norm(
lasagne.layers.DenseLayer(network2,
num_units=dense_units,
W=lasagne.init.GlorotUniform(),
name="dense1"))
network2 = batch_norm(
lasagne.layers.DenseLayer(network2,
num_units=parameters.N_USERS - 1,
W=lasagne.init.GlorotUniform(),
nonlinearity=lasagne.nonlinearities.softmax,
name="output"))
return network2
def cnn_sep(M, W1, W2, hh=.0001, ep=5000, d=0, sp=.0001, spb=3, al='rprop'):
# Facilitate reasonable convolutions core
theano.config.dnn.conv.algo_fwd = 'fft_tiling'
theano.config.dnn.conv.algo_bwd_filter = 'none'
theano.config.dnn.conv.algo_bwd_data = 'none'
# Reformat input data
M3 = reshape(M.astype(float32), (1, M.shape[0], M.shape[1]))
# Copy key variables to GPU
_M = theano.shared(M3.astype(float32))
# Get dictionary shapes
K = [W1.shape[1], W2.shape[1]]
T = W1.shape[2]
# We have weights to discover
H = theano.shared(
sqrt(2. / (K[0] + K[1] + M.shape[1])) *
random.rand(1, K[0] + K[1], M.T.shape[0]).astype(float32))
fI = InputLayer(shape=(1, K[0] + K[1], M.T.shape[0]), input_var=H)
# Split in two pathways
H1 = SliceLayer(fI, indices=slice(0, K[0]), axis=1)
H2 = SliceLayer(fI, indices=slice(K[0], K[0] + K[1]), axis=1)
# Compute source modulators using previously learned convolutional dictionaries
R1 = Conv1DLayer(H1,
filter_size=T,
W=W1,
num_filters=M.shape[0],
pad='same',
nonlinearity=lambda x: psoftplus(x, spb),
b=None)
R2 = Conv1DLayer(H2,
filter_size=T,
W=W2,
num_filters=M.shape[0],
pad='same',
nonlinearity=lambda x: psoftplus(x, spb),
b=None)
# Add the two approximations
R = ElemwiseSumLayer([R1, R2])
# Cost function
dum = Th.vector('dum')
Ro = get_output(R) + eps
cost = Th.mean(_M * (Th.log(_M + eps) - Th.log(Ro)) - _M +
Ro) + 0 * Th.mean(dum) + sp * Th.mean(abs(H))
# Train it using Lasagne
opt = downhill.build(al, loss=cost, inputs=[dum], params=[H])
train = downhill.Dataset(array([0]).astype(float32), batch_size=0)
er = downhill_train(opt, train, hh, ep, None)
# Get outputs
_r = squeeze(nget(R, dum, array([0]).astype(float32))) + eps
_r1 = squeeze(nget(R1, dum, array([0]).astype(float32)))
_r2 = squeeze(nget(R2, dum, array([0]).astype(float32)))
return _r, _r1, _r2, er
def cnn_fn(self, max_length):
"""Build the theano tensor
Using the attributes of the class, build the theano graph
using lasagne to get the neural networks which can
then be passed into theano.function to get the values
given input.
:param max_len: (int) max length of the language we are using
:return l_final: (tensor) theano tensor specifying the output of the cnn"""
l_in = InputLayer((None, max_length, self.vocab_size))
l_in_T = DimshuffleLayer(l_in, (0, 2, 1))
l_causal_conv = DilatedConv1DLayer(
l_in_T,
num_filters=self.nn_residual_channels,
dilation=1,
nonlinearity=None)
l_prev = l_causal_conv
skip_layers = []
for h in range(len(self.nn_dilations)):
l_filter = DilatedConv1DLayer(
l_prev,
num_filters=self.nn_dilation_channels,
dilation=self.nn_dilations[h],
nonlinearity=tanh)
l_gate = DilatedConv1DLayer(l_prev,
num_filters=self.nn_dilation_channels,
dilation=self.nn_dilations[h],
nonlinearity=sigmoid)
l_merge = ElemwiseMergeLayer([l_filter, l_gate],
merge_function=T.mul)
l_dense = Conv1DLayer(l_merge,
num_filters=self.nn_residual_channels,
filter_size=1,
nonlinearity=None)
l_residual = ElemwiseSumLayer([l_prev, l_dense])
l_skip = Conv1DLayer(l_merge,
num_filters=self.nn_residual_channels,
filter_size=1,
nonlinearity=None)
skip_layers.append(l_skip)
l_prev = l_residual
l_skip_sum = NonlinearityLayer(ElemwiseSumLayer(skip_layers),
nonlinearity=elu)
l_final = DimshuffleLayer(l_skip_sum, (0, 2, 1))
return l_final
def network_convpool_lstm_hybrid(input_vars,
nb_classes,
grad_clip=110,
imsize=[32, 32],
n_colors=3,
n_timewin=5,
n_layers=(4, 2),
n_filters_first=32,
dense_num_unit=[512, 512],
shared_weights=True,
batch_norm_dense=False,
batch_norm_conv=False):
"""
Builds the complete network with LSTM and 1D-conv layers combined
:param input_vars: list of EEG images (one image per time window)
:param nb_classes: number of classes
:param grad_clip: the gradient messages are clipped to the given value during
the backward pass.
:param imsize: size of the input image (assumes a square input)
:param n_colors: number of color channels in the image
:param n_timewin: number of time windows in the snippet
:return: a pointer to the output of last layer
"""
convnets_lstm = []
convnets_first = []
n_layers_first = [1]
w_init = None
# Build 5 parallel CNNs with shared weights
for i in range(n_timewin):
if i == 0:
convnet, w_init = build_cnn(input_vars[i],
imsize=imsize,
n_colors=n_colors,
n_filters_first=n_filters_first,
n_layers=n_layers,
isMaxpool=False,
filter_size=(2, 2),
batch_norm_conv=batch_norm_conv)
if not shared_weights:
w_init = None
else:
convnet, _ = build_cnn(input_vars[i],
w_init=w_init,
imsize=imsize,
n_colors=n_colors,
n_filters_first=n_filters_first,
n_layers=n_layers,
isMaxpool=False,
filter_size=(2, 2),
batch_norm_conv=batch_norm_conv)
convnets_lstm.append(FlattenLayer(convnet))
convnets_first.append(convnet)
# build lstm network
# at this point convnets shape is [numTimeWin][n_samples, features]
# we want the shape to be [n_samples, features, numTimeWin]
convpool = ConcatLayer(convnets_lstm)
convpool = ReshapeLayer(
convpool, ([0], n_timewin, get_output_shape(convnets_lstm[0])[1]))
# Input to LSTM should have the shape as (batch size, SEQ_LENGTH, num_features)
lstm = LSTMLayer(convpool,
num_units=128,
grad_clipping=grad_clip,
nonlinearity=lasagne.nonlinearities.tanh)
lstm_out = SliceLayer(lstm, -1, 1)
# Build Second 5 parallel CNNs with shared weights
w_init = None
n_filters_second = 32
n_layers_second = (3, 2)
convnets_second = []
for i in range(n_timewin):
if i == 0:
convnet, w_init = build_cnn_int(convnets_first[i],
n_filters_first=n_filters_second,
n_layers=n_layers_second)
else:
convnet, _ = build_cnn_int(convnets_first[i],
w_init=w_init,
n_filters_first=n_filters_second,
n_layers=n_layers_second)
convnets_second.append(FlattenLayer(convnet))
# at this point convnets shape is [numTimeWin][n_samples, features]
# we want the shape to be [n_samples, features, numTimeWin]
convpool2 = ConcatLayer(convnets_second)
convpool2 = ReshapeLayer(
convpool2, ([0], n_timewin, get_output_shape(convnets_second[0])[1]))
convpool2 = DimshuffleLayer(convpool2, (0, 2, 1))
# input to 1D convlayer should be in (batch_size, num_input_channels, input_length)
convpool2 = Conv1DLayer(convpool2, 64, 3)
convpool2 = FlattenLayer(convpool2)
# Merge 1D-Conv and LSTM outputs
dense_input = ConcatLayer([convpool2, lstm_out])
# A fully-connected layer of 256 units with 50% dropout on its inputs:
for i in range(len(dense_num_unit)):
convpool = DenseLayer(lasagne.layers.dropout(dense_input, p=.5),
num_units=dense_num_unit[i],
nonlinearity=lasagne.nonlinearities.rectify)
if batch_norm_dense:
convpool = batch_norm(convpool)
# And, finally, the 10-unit output layer with 50% dropout on its inputs:
if nb_classes == 1:
nonlinearity = lasagne.nonlinearities.sigmoid
else:
nonlinearity = lasagne.nonlinearities.softmax
convpool = DenseLayer(lasagne.layers.dropout(convpool, p=.5),
num_units=nb_classes,
nonlinearity=nonlinearity)
return convpool
def network_convpool_conv1d(input_vars,
nb_classes,
imsize=[32, 32],
n_colors=3,
n_timewin=5,
n_layers=(4, 2),
n_filters_first=32,
dense_num_unit=[512, 512],
shared_weights=True,
batch_norm_dense=False,
batch_norm_conv=False):
"""
Builds the complete network with 1D-conv layer to integrate time from sequences of EEG images.
:param input_vars: list of EEG images (one image per time window)
:param nb_classes: number of classes
:param imsize: size of the input image (assumes a square input)
:param n_colors: number of color channels in the image
:param n_timewin: number of time windows in the snippet
:return: a pointer to the output of last layer
"""
convnets = []
w_init = None
# Build 5 parallel CNNs with shared weights
for i in range(n_timewin):
if i == 0:
convnet, w_init = build_cnn(input_vars[i],
imsize=imsize,
n_colors=n_colors,
n_filters_first=n_filters_first,
n_layers=n_layers,
batch_norm_conv=batch_norm_conv)
if not shared_weights:
w_init = None
else:
convnet, _ = build_cnn(input_vars[i],
w_init=w_init,
imsize=imsize,
n_colors=n_colors,
n_filters_first=n_filters_first,
n_layers=n_layers,
batch_norm_conv=batch_norm_conv)
convnets.append(FlattenLayer(convnet))
# at this point convnets shape is [numTimeWin][n_samples, features]
# we want the shape to be [n_samples, features, numTimeWin]
convpool = ConcatLayer(convnets)
convpool = ReshapeLayer(convpool,
([0], n_timewin, get_output_shape(convnets[0])[1]))
convpool = DimshuffleLayer(convpool, (0, 2, 1))
# input to 1D convlayer should be in (batch_size, num_input_channels, input_length)
convpool = Conv1DLayer(convpool, 64, 3)
convpool = MaxPool1DLayer(convpool, pool_size=(2))
convpool = Conv1DLayer(convpool, 128, 3)
# A fully-connected layer of 512 units with 50% dropout on its inputs:
for i in range(len(dense_num_unit)):
convpool = DenseLayer(lasagne.layers.dropout(convpool, p=.5),
num_units=dense_num_unit[i],
nonlinearity=lasagne.nonlinearities.rectify)
if batch_norm_dense:
convpool = batch_norm(convpool)
# And, finally, the output layer with 50% dropout on its inputs:
if nb_classes == 1:
nonlinearity = lasagne.nonlinearities.sigmoid
else:
nonlinearity = lasagne.nonlinearities.softmax
convpool = DenseLayer(lasagne.layers.dropout(convpool, p=.5),
num_units=nb_classes,
nonlinearity=nonlinearity)
return convpool
def nn_fn(self):
l_in_x = InputLayer((None, self.embedding_dim, self.max_length))
l_in_z = InputLayer((None, self.z_dim))
l_causal_conv = DilatedConv1DLayer(
l_in_x,
num_filters=self.nn_residual_channels,
dilation=1,
nonlinearity=None)
l_prev = l_causal_conv
skip_layers = []
for h in range(len(self.nn_dilations)):
l_x_filter = DilatedConv1DLayer(
l_prev,
num_filters=self.nn_dilation_channels,
dilation=self.nn_dilations[h],
nonlinearity=None,
b=None)
l_z_filter = DenseLayer(l_in_z,
num_units=self.nn_dilation_channels,
nonlinearity=None,
b=None)
l_z_filter_reshape = ReshapeLayer(l_z_filter, (
[0],
[1],
1,
))
l_z_filter_rep = RepeatLayer(l_z_filter_reshape,
self.max_length,
axis=-1,
ndim=3)
l_filter = NonlinearityLayer(ElemwiseSumLayer(
[l_x_filter, l_z_filter_rep]),
nonlinearity=tanh)
l_x_gate = DilatedConv1DLayer(
l_prev,
num_filters=self.nn_dilation_channels,
dilation=self.nn_dilations[h],
nonlinearity=None,
b=None)
l_z_gate = DenseLayer(l_in_z,
num_units=self.nn_dilation_channels,
nonlinearity=None,
b=None)
l_z_gate_reshape = ReshapeLayer(l_z_gate, (
[0],
[1],
1,
))
l_z_gate_rep = RepeatLayer(l_z_gate_reshape,
self.max_length,
axis=-1,
ndim=3)
l_gate = NonlinearityLayer(ElemwiseSumLayer(
[l_x_gate, l_z_gate_rep]),
nonlinearity=sigmoid)
l_merge = ElemwiseMergeLayer([l_filter, l_gate],
merge_function=T.mul)
l_dense = Conv1DLayer(l_merge,
num_filters=self.nn_residual_channels,
filter_size=1,
nonlinearity=None,
b=None)
l_residual = ElemwiseSumLayer([l_prev, l_dense])
l_skip = Conv1DLayer(l_merge,
num_filters=self.embedding_dim,
filter_size=1,
nonlinearity=None,
b=None)
skip_layers.append(l_skip)
l_prev = l_residual
l_skip_sum = NonlinearityLayer(ElemwiseSumLayer(skip_layers),
nonlinearity=elu)
l_prev = l_skip_sum
for h in range(2):
l_h = Conv1DLayer(l_prev,
num_filters=self.embedding_dim,
filter_size=1,
nonlinearity=None,
b=None)
l_z = DenseLayer(l_in_z,
num_units=self.embedding_dim,
nonlinearity=None,
b=None)
l_z_reshape = ReshapeLayer(l_z, (
[0],
[1],
1,
))
l_z_reshape_rep = RepeatLayer(l_z_reshape,
self.max_length,
axis=-1,
ndim=3)
l_sum = NonlinearityLayer(ElemwiseSumLayer([l_h, l_z_reshape_rep]),
nonlinearity=elu)
l_prev = l_sum
l_out = DimshuffleLayer(l_prev, (0, 2, 1))
return (l_in_x, l_in_z), l_out
