def __init__(self, input_shape, output_dim, hidden_sizes,
conv_filters, conv_filter_sizes, conv_strides, conv_pads,
hidden_W_init=LI.GlorotUniform(), hidden_b_init=LI.Constant(0.),
output_W_init=LI.GlorotUniform(), output_b_init=LI.Constant(0.),
# conv_W_init=LI.GlorotUniform(), conv_b_init=LI.Constant(0.),
hidden_nonlinearity=LN.rectify,
output_nonlinearity=LN.softmax,
name=None, input_var=None):
if name is None:
prefix = ""
else:
prefix = name + "_"
if len(input_shape) == 3:
l_in = L.InputLayer(shape=(None, np.prod(input_shape)), input_var=input_var)
l_hid = L.reshape(l_in, ([0],) + input_shape)
elif len(input_shape) == 2:
l_in = L.InputLayer(shape=(None, np.prod(input_shape)), input_var=input_var)
input_shape = (1,) + input_shape
l_hid = L.reshape(l_in, ([0],) + input_shape)
else:
l_in = L.InputLayer(shape=(None,) + input_shape, input_var=input_var)
l_hid = l_in
for idx, conv_filter, filter_size, stride, pad in izip(
xrange(len(conv_filters)),
conv_filters,
conv_filter_sizes,
conv_strides,
conv_pads,
):
l_hid = L.Conv2DLayer(
l_hid,
num_filters=conv_filter,
filter_size=filter_size,
stride=(stride, stride),
pad=pad,
nonlinearity=hidden_nonlinearity,
name="%sconv_hidden_%d" % (prefix, idx),
convolution=wrapped_conv,
)
for idx, hidden_size in enumerate(hidden_sizes):
l_hid = L.DenseLayer(
l_hid,
num_units=hidden_size,
nonlinearity=hidden_nonlinearity,
name="%shidden_%d" % (prefix, idx),
W=hidden_W_init,
b=hidden_b_init,
)
l_out = L.DenseLayer(
l_hid,
num_units=output_dim,
nonlinearity=output_nonlinearity,
name="%soutput" % (prefix,),
W=output_W_init,
b=output_b_init,
)
self._l_in = l_in
self._l_out = l_out
self._input_var = l_in.input_var
def build_model():
#################
# Regular model #
#################
l0 = InputLayer(data_sizes["sliced:data:ax"])
l0r = reshape(l0, (
-1,
1,
) + data_sizes["sliced:data:ax"][-2:])
# first do the segmentation steps
l1a = ConvLayer(
l0r,
num_filters=32,
filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1b = ConvLayer(
l1a,
num_filters=32,
filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1c = ConvLayer(
l1b,
num_filters=64,
filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1f = ConvLayer(l1c,
num_filters=1,
filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
nonlinearity=lasagne.nonlinearities.sigmoid)
l_1r = reshape(l1f, data_sizes["sliced:data:ax"])
l_d3 = lasagne.layers.DenseLayer(l_1r, num_units=2)
l_systole = MuLogSigmaErfLayer(l_d3)
l_d3b = lasagne.layers.DenseLayer(l_1r, num_units=2)
l_diastole = MuLogSigmaErfLayer(l_d3b)
return {
"inputs": {
"sliced:data:ax": l0,
},
"outputs": {
"systole": l_systole,
"diastole": l_diastole
}
}
def build_cnn():
data_size = (None, 10, 100) # Batch size x Img Channels x Height x Width
input_var = T.tensor3(name="input", dtype='int64')
values = np.array(np.random.randint(0, 1, (5, 10, 100)))
input_var.tag.test_value = values
input_layer = L.InputLayer(data_size, input_var=input_var)
W = create_char_embedding_matrix()
embed_layer = L.EmbeddingLayer(input_layer,
input_size=102,
output_size=101,
W=W)
reshape = L.reshape(embed_layer, (-1, 100, 101))
dim_shuffle = L.dimshuffle(reshape, (0, 2, 1))
#conv_layer_1 = L.Conv2DLayer(embed_layer, 4, (1), 1, 0)
#pool_layer_1 = L.MaxPool1DLayer(conv_layer_1, pool_size=1)
print L.get_output(dim_shuffle).tag.test_value.shape
conv_layer_1 = L.Conv1DLayer(dim_shuffle, 50, 2, 1)
print L.get_output(conv_layer_1).tag.test_value.shape
print "TEST"
pool_layer_1 = L.MaxPool1DLayer(conv_layer_1, pool_size=99)
print L.get_output(pool_layer_1).tag.test_value.shape
reshape_conv_1 = L.reshape(pool_layer_1, (-1, 50))
conv_layer_2 = L.Conv1DLayer(dim_shuffle, 50, 3, 1)
pool_layer_2 = L.MaxPool1DLayer(conv_layer_2, pool_size=98)
reshape_conv_2 = L.reshape(pool_layer_2, (-1, 50))
merge_layer = L.ConcatLayer([reshape_conv_1, reshape_conv_2], 1)
print L.get_output(merge_layer).tag.test_value.shape
reshape_output = L.reshape(merge_layer, (-1, 10, 100))
print L.get_output(reshape_output).tag.test_value.shape
x = T.tensor3(name="testname", dtype='int32')
#x = T.imatrix()
#output = L.get_output(conv_layer_1,x)
#f = theano.function([x],output)
word = unicode("Tat")
word_index = np.array([])
#print word_index
#x_test = np.array([word_index]).astype('int32')
#print f(x_test)
return reshape_output
def _remove_all_first_tokens(input_layer, unflatten_sequences_shape, flatten_input=True):
"""
Helper function that creates a sequence of layers to clean up the tensor from all elements,
corresponding to the first token in the sequence
"""
new_flattened_shape = (unflatten_sequences_shape[0] * (unflatten_sequences_shape[1] - 1),) \
+ unflatten_sequences_shape[2:]
sliced = SliceLayer(
incoming=reshape(input_layer, unflatten_sequences_shape) if flatten_input else input_layer,
indices=slice(1, None),
axis=1) # sequence axis
return reshape(sliced, new_flattened_shape)
def build_model():
#################
# Regular model #
#################
l0 = InputLayer(data_sizes["sliced:data:ax"])
l0r = reshape(l0, (-1, 1, ) + data_sizes["sliced:data:ax"][-2:])
# first do the segmentation steps
l1a = ConvLayer(l0r, num_filters=32, filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1b = ConvLayer(l1a, num_filters=32, filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1c = ConvLayer(l1b, num_filters=64, filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1f = ConvLayer(l1c, num_filters=1, filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
nonlinearity=lasagne.nonlinearities.sigmoid)
l_1r = reshape(l1f, data_sizes["sliced:data:ax"])
l_d3 = lasagne.layers.DenseLayer(l_1r,
num_units=2,
nonlinearity=lasagne.nonlinearities.identity)
l_systole = MuLogSigmaErfLayer(l_d3)
l_d3b = lasagne.layers.DenseLayer(l_1r,
num_units=2,
nonlinearity=lasagne.nonlinearities.identity)
l_diastole = MuLogSigmaErfLayer(l_d3b)
return {
"inputs":{
"sliced:data:ax": l0
},
"outputs":{
"systole": l_systole,
"diastole": l_diastole
}
}
def build_cnn(input):
#data_size = (None,103,130) # Batch size x Img Channels x Height x Width
#input_var = T.tensor3(name = "input",dtype='int64')
input_var = input
#values = np.array(np.random.randint(0,102,(1,9,50)))
#input_var.tag.test_value = values
#number sentences x words x characters
input_layer = L.InputLayer((None,9,50), input_var=input)
W = create_char_embedding_matrix()
embed_layer = L.EmbeddingLayer(input_layer, input_size=103,output_size=101, W=W)
#print "EMBED", L.get_output(embed_layer).tag.test_value.shape
reshape_embed = L.reshape(embed_layer,(-1,50,101))
#print "reshap embed", L.get_output(reshape_embed).tag.test_value.shape
conv_layer_1 = L.Conv1DLayer(reshape_embed, 55, 2)
conv_layer_2 = L.Conv1DLayer(reshape_embed, 55, 3)
#print "TEST"
#print "Convolution Layer 1", L.get_output(conv_layer_1).tag.test_value.shape
#print "Convolution Layer 2", L.get_output(conv_layer_2).tag.test_value.shape
#flatten_conv_1 = L.flatten(conv_layer_1,3)
#flatten_conv_2 = L.flatten(conv_layer_2,3)
#reshape_max_1 = L.reshape(flatten_conv_1,(-1,49))
#reshape_max_2 = L.reshape(flatten_conv_2, (-1,48))
#print "OUTPUT Flatten1", L.get_output(flatten_conv_1).tag.test_value.shape
#print "OUTPUT Flatten2", L.get_output(flatten_conv_2).tag.test_value.shape
#print "OUTPUT reshape_max_1", L.get_output(reshape_max_1).tag.test_value.shape
#print "OUTPUT reshape_max_2", L.get_output(reshape_max_2).tag.test_value.shape
pool_layer_1 = L.MaxPool1DLayer(conv_layer_1, pool_size=54)
pool_layer_2 = L.MaxPool1DLayer(conv_layer_2, pool_size=53)
#print "OUTPUT POOL1", L.get_output(pool_layer_1).tag.test_value.shape
#print "OUTPUT POOL2",L.get_output(pool_layer_2).tag.test_value.shape
merge_layer = L.ConcatLayer([pool_layer_1, pool_layer_2], 1)
flatten_merge = L.flatten(merge_layer, 2)
reshape_merge = L.reshape(flatten_merge, (1,9,110))
print L.get_output(reshape_embed).shape
#print L.get_output(reshape_merge).tag.test_value.shape
return reshape_merge, char_index_lookup
def build_net(nz=10):
# nz = size of latent code
#N.B. using batch_norm applies bn before non-linearity!
F = 32
enc = InputLayer(shape=(None, 1, 28, 28))
enc = Conv2DLayer(incoming=enc,
num_filters=F * 2,
filter_size=5,
stride=2,
nonlinearity=lrelu(0.2),
pad=2)
enc = Conv2DLayer(incoming=enc,
num_filters=F * 4,
filter_size=5,
stride=2,
nonlinearity=lrelu(0.2),
pad=2)
enc = Conv2DLayer(incoming=enc,
num_filters=F * 4,
filter_size=5,
stride=1,
nonlinearity=lrelu(0.2),
pad=2)
enc = reshape(incoming=enc, shape=(-1, F * 4 * 7 * 7))
enc = DenseLayer(incoming=enc, num_units=nz, nonlinearity=sigmoid)
#Generator networks
dec = InputLayer(shape=(None, nz))
dec = DenseLayer(incoming=dec, num_units=F * 4 * 7 * 7)
dec = reshape(incoming=dec, shape=(-1, F * 4, 7, 7))
dec = Deconv2DLayer(incoming=dec,
num_filters=F * 4,
filter_size=4,
stride=2,
nonlinearity=relu,
crop=1)
dec = Deconv2DLayer(incoming=dec,
num_filters=F * 4,
filter_size=4,
stride=2,
nonlinearity=relu,
crop=1)
dec = Deconv2DLayer(incoming=dec,
num_filters=1,
filter_size=3,
stride=1,
nonlinearity=sigmoid,
crop=1)
return enc, dec
def _add_context_encoder(self):
self._net['batched_enc'] = reshape(
self._net['enc'], (self._batch_size, self._input_context_size,
get_output_shape(self._net['enc'])[-1]))
self._net['context_enc'] = GRULayer(incoming=self._net['batched_enc'],
num_units=self._hidden_layer_dim,
grad_clipping=self._grad_clip,
only_return_final=True,
name='context_encoder')
self._net['switch_enc_to_tv'] = T.iscalar(name='switch_enc_to_tv')
self._net['thought_vector'] = InputLayer(
shape=(None, self._hidden_layer_dim),
input_var=T.fmatrix(name='thought_vector'),
name='thought_vector')
self._net['enc_result'] = SwitchLayer(
incomings=[self._net['thought_vector'], self._net['context_enc']],
condition=self._net['switch_enc_to_tv'])
# We need the following to pass as 'givens' argument when compiling theano functions:
self._default_thoughts_vector = T.zeros(
(self._batch_size, self._hidden_layer_dim))
self._default_input_x = T.zeros(
shape=(self._net['thought_vector'].input_var.shape[0], 1, 1),
dtype=np.int32)
def broadcast_dot_layer(l_pred, l_targets, feature_dim, id_tag):
l_broadcast = dimshuffle(l_pred, (0, 1, 'x'), name=id_tag + 'dot_broadcast')
l_forget = ForgetSizeLayer(l_broadcast, axis=2, name=id_tag + 'dot_nosize')
l_merge = ElemwiseMergeLayer((l_forget, l_targets), T.mul, name=id_tag + 'dot_elemwise_mul')
l_pool = FeaturePoolLayer(l_merge, pool_size=feature_dim, axis=1,
pool_function=T.sum, name=id_tag + 'dot_pool')
return reshape(l_pool, ([0], [2]), name=id_tag + 'broadcast_dot')
def make_broadcasted_heads(incoming,head_size,name=None):
ndim = len(incoming.output_shape)
assert ndim in (2,3), "incoming must be 2-dimensional (query) or 3-dimensional (key or value)"
heads = DenseLayer(incoming,head_size*num_heads,nonlinearity=None,
num_leading_axes=ndim-1,name=name) #[batch,time,head_size*num_heads]
if ndim == 3:
heads = reshape(heads,([0],[1],head_size,num_heads), name=name) #[batch,time,head_size,num_heads]
broadcasted_heads = BroadcastLayer(heads, (0, 3), name=name) #[batch*heads,time,head_size]
else: #ndim == 2
heads = reshape(heads, ([0], head_size, num_heads), name=name) # [batch,head_size,num_heads]
broadcasted_heads = BroadcastLayer(heads, (0, 2), name=name) # [batch*heads, head_size]
return broadcasted_heads
def build_sequence_summarizing_layer(input_var, input_layer, num_units,
num_layers, output_dim):
# input_layer: n_batch, n_time_steps, n_feat
n_batch, n_time_steps, _ = input_var.shape
# n_batch, n_feat
prev_layer = reshape(input_layer, (-1, [2]))
for i in range(num_layers):
prev_layer = DenseLayer(prev_layer,
num_units=num_units,
nonlinearity=nonlinearities.tanh)
dense_layer = DenseLayer(prev_layer,
num_units=output_dim,
nonlinearity=nonlinearities.identity)
return SummarizingLayer(
reshape(dense_layer, (n_batch, n_time_steps, output_dim)))
def score_fused_convnets(self,
fusion_type,
input_var1=None,
input_var2=None,
weights_dir_depth=None,
weights_dir_rgb=None,
bottleneck_W=None,
weights_dir=None):
net = OrderedDict()
rgb_net = self.simple_convnet(4,
input_var=input_var1,
bottleneck_W=bottleneck_W)
depth_net = self.simple_convnet(1,
input_var=input_var2,
bottleneck_W=bottleneck_W)
if weights_dir_depth is not None and weights_dir_rgb is not None:
lw_depth = LoadWeights(weights_dir_depth, depth_net)
lw_depth.load_weights_numpy()
lw_rgb = LoadWeights(weights_dir_rgb, rgb_net)
lw_rgb.load_weights_numpy()
if fusion_type == self.LOCAL:
net['reshape_depth'] = reshape(depth_net['output'],
([0], 1, 1, [1]))
net['reshape_rgb'] = reshape(rgb_net['output'], ([0], 1, 1, [1]))
net['concat'] = concat([net['reshape_depth'], net['reshape_rgb']])
net['lcl'] = LocallyConnected2DLayer(net['concat'],
1, (1, 1),
untie_biases=True,
nonlinearity=None)
net['output'] = reshape(net['lcl'], ([0], [3]))
elif fusion_type == self.SUM:
net['output'] = ElemwiseSumLayer(
[depth_net['output'], rgb_net['output']], coeffs=0.5)
if weights_dir is not None:
lw = LoadWeights(weights_dir, net)
lw.load_weights_numpy()
return net
def build_network(W,
number_unique_tags,
longest_word,
longest_sentence,
input_var=None):
print("Building network ...")
input_layer = L.InputLayer((None, longest_sentence, longest_word),
input_var=input_var)
embed_layer = L.EmbeddingLayer(input_layer,
input_size=103,
output_size=101,
W=W)
reshape_embed = L.reshape(embed_layer, (-1, longest_word, 101))
conv_layer_1 = L.Conv1DLayer(reshape_embed, longest_word, 2)
conv_layer_2 = L.Conv1DLayer(reshape_embed, longest_word, 3)
pool_layer_1 = L.MaxPool1DLayer(conv_layer_1, pool_size=longest_word - 1)
pool_layer_2 = L.MaxPool1DLayer(conv_layer_2, pool_size=longest_word - 2)
merge_layer = L.ConcatLayer([pool_layer_1, pool_layer_2], 1)
flatten_merge = L.flatten(merge_layer, 2)
reshape_merge = L.reshape(flatten_merge,
(-1, longest_sentence, int(longest_word * 2)))
l_re = lasagne.layers.RecurrentLayer(
reshape_merge,
N_HIDDEN,
nonlinearity=lasagne.nonlinearities.sigmoid,
mask_input=None)
l_out = lasagne.layers.DenseLayer(
l_re, number_unique_tags, nonlinearity=lasagne.nonlinearities.softmax)
print "DONE BUILDING NETWORK"
return l_out
def __init__(self, input_shape, output_dim, servoing_pol,
name=None, input_var=None):
if name is None:
prefix = ""
else:
prefix = name + "_"
if len(input_shape) == 3:
l_in = L.InputLayer(shape=(None, np.prod(input_shape)), input_var=input_var)
l_hid = L.reshape(l_in, ([0],) + input_shape)
elif len(input_shape) == 2:
l_in = L.InputLayer(shape=(None, np.prod(input_shape)), input_var=input_var)
input_shape = (1,) + input_shape
l_hid = L.reshape(l_in, ([0],) + input_shape)
else:
l_in = L.InputLayer(shape=(None,) + input_shape, input_var=input_var)
l_hid = l_in
l_out = TheanoServoingPolicyLayer(l_hid, servoing_pol)
self._l_in = l_in
self._l_out = l_out
self._input_var = l_in.input_var
def _add_word_embeddings(self):
self._net['input_x'] = InputLayer(shape=(None, None, None),
input_var=T.itensor3(name='input_x'),
name='input_x')
self._net['input_y'] = InputLayer(shape=(None, None),
input_var=T.imatrix(name='input_y'),
name='input_y')
# Infer these variables from data passed to computation graph since batch shape may differ in training and
# prediction phases
self._batch_size = self._net['input_x'].input_var.shape[0]
self._input_context_size = self._net['input_x'].input_var.shape[1]
self._input_seq_len = self._net['input_x'].input_var.shape[2]
self._output_seq_len = self._net['input_y'].input_var.shape[1]
self._net['input_x_batched'] = \
reshape(self._net['input_x'], (self._batch_size * self._input_context_size, self._input_seq_len))
self._net['input_x_mask'] = NotEqualMaskLayer(
incoming=self._net['input_x_batched'],
x=self._skip_token_id,
name='mask_x')
self._net['emb_x'] = EmbeddingLayer(
incoming=self._net['input_x_batched'],
input_size=self._vocab_size,
output_size=self._word_embedding_dim,
W=self._W_init_embedding,
name='emb_x')
# output shape (batch_size, input_context_size, input_seq_len, embedding_dimension)
self._net['input_y_mask'] = NotEqualMaskLayer(
incoming=self._net['input_y'],
x=self._skip_token_id,
name='mask_y')
self._net['emb_y'] = EmbeddingLayer(
incoming=self._net['input_y'],
input_size=self._vocab_size,
output_size=self._word_embedding_dim,
W=self._W_init_embedding,
name='emb_y')
# output shape (batch_size, output_seq_len, embedding_dimension)
if not self._train_word_embedding:
self._net['emb_x'].params[self._net['emb_x'].W].remove('trainable')
self._net['emb_y'].params[self._net['emb_y'].W].remove('trainable')
def get_input_layer(self, input_vars, recurrent_length=0, cell_size=20, context_len=1, id=None):
id_tag = (id + '/') if id else ''
(input_var,) = input_vars
shape = ((None, context_len * len(self.buckets))
if recurrent_length == 0 else
(None, recurrent_length, context_len * len(self.buckets)))
l_color = InputLayer(shape=shape, input_var=input_var,
name=id_tag + 'color_input')
l_color_embed = EmbeddingLayer(l_color, input_size=sum(b.num_types for b in self.buckets),
output_size=cell_size,
name=id_tag + 'color_embed')
dims = (([0], -1) if recurrent_length == 0 else ([0], [1], -1))
l_color_flattened = reshape(l_color_embed, dims)
return l_color_flattened, [l_color]
def _add_word_embeddings(self):
self._net['input_x'] = InputLayer(shape=(None, None, None),
input_var=T.itensor3(name='input_x'),
name='input_x')
self._net['input_y'] = InputLayer(shape=(None, None),
input_var=T.imatrix(name='input_y'),
name='input_y')
# These are theano variables and they are computed dynamically as data is passed into the computational graph
self._batch_size = self._net['input_x'].input_var.shape[0]
self._input_context_size = self._net['input_x'].input_var.shape[1]
self._input_seq_len = self._net['input_x'].input_var.shape[2]
self._output_seq_len = self._net['input_y'].input_var.shape[1]
self._net['input_x_batched'] = \
reshape(self._net['input_x'], (self._batch_size * self._input_context_size, self._input_seq_len))
self._net['input_x_mask'] = NotEqualMaskLayer(
incoming=self._net['input_x_batched'],
x=self._skip_token_id,
name='mask_x')
self._net['emb_x'] = EmbeddingLayer(
incoming=self._net['input_x_batched'],
input_size=self._vocab_size,
output_size=self._word_embedding_dim,
W=self._W_init_embedding,
name='emb_x')
# output shape (batch_size, input_context_size, input_seq_len, embedding_dimension)
self._net['input_y_mask'] = NotEqualMaskLayer(
incoming=self._net['input_y'],
x=self._skip_token_id,
name='mask_y')
self._net['emb_y'] = EmbeddingLayer(
incoming=self._net['input_y'],
input_size=self._vocab_size,
output_size=self._word_embedding_dim,
W=self._W_init_embedding,
name='emb_y')
# output shape (batch_size, output_seq_len, embedding_dimension)
if not self._train_word_embedding:
self._net['emb_x'].params[self._net['emb_x'].W].remove('trainable')
self._net['emb_y'].params[self._net['emb_y'].W].remove('trainable')
def get_input_layer(self, input_vars, recurrent_length=0, cell_size=20,
context_len=1, id=None):
id_tag = (id + '/') if id else ''
(input_var,) = input_vars
input_shape = ((None, context_len)
if recurrent_length == 0 else
(None, recurrent_length, context_len))
l_color = InputLayer(shape=input_shape, input_var=input_var,
name=id_tag + 'color_input')
l_color_embed = EmbeddingLayer(l_color, input_size=self.num_types,
output_size=cell_size,
name=id_tag + 'color_embed')
output_shape = (([0], context_len * cell_size)
if recurrent_length == 0 else
([0], recurrent_length, context_len * cell_size))
l_color_shape = reshape(l_color_embed, output_shape, name=id_tag + 'color_embed_flattened')
return l_color_shape, [l_color]
def get_input_layer(self, input_vars, recurrent_length=0, cell_size=20, context_len=1, id=None):
id_tag = (id + '/') if id else ''
(input_var,) = input_vars
context_repr_size = None if context_len is None else len(self.buckets) * context_len
shape = ((None, context_repr_size)
if recurrent_length == 0 else
(None, recurrent_length, context_repr_size))
l_color = InputLayer(shape=shape, input_var=input_var,
name=id_tag + 'color_input')
l_color_embed = EmbeddingLayer(l_color, input_size=sum(b.num_types for b in self.buckets),
output_size=cell_size,
name=id_tag + 'color_embed')
dims = (([0], -1) if recurrent_length == 0 else ([0], [1], -1))
l_color_flattened = reshape(l_color_embed, dims)
return l_color_flattened, [l_color]
def get_input_layer(self, input_vars, recurrent_length=0, cell_size=20,
context_len=1, id=None):
id_tag = (id + '/') if id else ''
(input_var,) = input_vars
input_shape = ((None, context_len)
if recurrent_length == 0 else
(None, recurrent_length, context_len))
l_color = InputLayer(shape=input_shape, input_var=input_var,
name=id_tag + 'color_input')
l_color_embed = EmbeddingLayer(l_color, input_size=self.num_types,
output_size=cell_size,
name=id_tag + 'color_embed')
context_repr_size = -1 if context_len is None else context_len * cell_size
output_shape = (([0], context_repr_size)
if recurrent_length == 0 else
([0], recurrent_length, context_repr_size))
l_color_shape = reshape(l_color_embed, output_shape, name=id_tag + 'color_embed_flattened')
return l_color_shape, [l_color]
def _add_output_dense(self):
"""
Adds a dense layer on top of the decoder to convert hidden_state vector to probs distribution over vocabulary.
For every prob sequence last prob vectors are cut off since they correspond
to the tokens that go after EOS_TOKEN and we are not interested in them.
Doesn't need to reshape back the cut tensor since it's convenient to compare
this "long" output with true one-hot vectors.
"""
self._net['dec_dropout'] = DropoutLayer(incoming=reshape(
self._net['dec'], (-1, self._hidden_layer_dim)),
p=self._dense_dropout_ratio,
name='decoder_dropout_layer')
self._net['target'] = self._remove_all_first_tokens(
self._net['input_y'],
unflatten_sequences_shape=(self._batch_size, self._output_seq_len),
flatten_input=False)
self._net['dec_dropout_nolast'] = self._remove_all_last_tokens(
self._net['dec_dropout'],
unflatten_sequences_shape=(self._batch_size, self._output_seq_len,
self._hidden_layer_dim))
self._net['dist_nolast'] = DenseLayer(
incoming=self._net['dec_dropout_nolast'],
num_units=self._vocab_size,
nonlinearity=lasagne.nonlinearities.softmax,
name='dense_output_probs')
dist_layer_params = get_all_params(self._net['dist_nolast'])
param_name_to_param = {p.name: p for p in dist_layer_params}
self._net['dist'] = DenseLayer(
incoming=self._net['dec_dropout'],
num_units=self._vocab_size,
nonlinearity=lasagne.nonlinearities.softmax,
W=param_name_to_param['dense_output_probs.W'],
b=param_name_to_param['dense_output_probs.b'],
name='dense_output_probs')
def build_model():
#################
# Regular model #
#################
l0 = InputLayer(data_sizes["sliced:data:ax"])
l0r = reshape(l0, (-1, 1, ) + data_sizes["sliced:data:ax"][1:])
# (batch, channel, time, axis, x, y)
# convolve over time
l1 = ConvolutionOverAxisLayer(l0r, num_filters=2, filter_size=(5,), axis=(2,), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1m = MaxPoolOverAxisLayer(l1, pool_size=(2,), axis=(2,))
# convolve over x and y
l2a = ConvolutionOver2DAxisLayer(l1m, num_filters=32, filter_size=(5, 5), stride=(2,2),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l2b = ConvolutionOver2DAxisLayer(l2a, num_filters=32, filter_size=(5, 5),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l2m = MaxPoolOver2DAxisLayer(l2b, pool_size=(3, 3), axis=(4,5))
# convolve over x, y and axis
l3a = ConvolutionOver3DAxisLayer(l2m, num_filters=32, filter_size=(3, 3, 3),
axis=(3,4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l3b = ConvolutionOver3DAxisLayer(l3a, num_filters=32, filter_size=(3, 3, 3),
axis=(3,4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l3m = MaxPoolOver3DAxisLayer(l3b, pool_size=(2, 2, 2), axis=(3,4,5))
# convolve over time
l4 = ConvolutionOverAxisLayer(l3m, num_filters=32, filter_size=(5,), axis=(2,), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l4m = MaxPoolOverAxisLayer(l4, pool_size=(2,), axis=(2,))
# convolve over axis
l5 = ConvolutionOverAxisLayer(l4m, num_filters=32, filter_size=(3,), axis=(3,), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
"""
l_d3 = lasagne.layers.DenseLayer(l3m,
num_units=2,
nonlinearity=lasagne.nonlinearities.identity)
l_systole = MuLogSigmaErfLayer(l_d3)
l_d3b = lasagne.layers.DenseLayer(l3m,
num_units=2,
nonlinearity=lasagne.nonlinearities.identity)
l_diastole = MuLogSigmaErfLayer(l_d3b)
"""
l_d3 = lasagne.layers.DenseLayer(l5,
num_units=600,
nonlinearity=lasagne.nonlinearities.softmax)
l_systole = CumSumLayer(l_d3)
l_d3b = lasagne.layers.DenseLayer(l5,
num_units=600,
nonlinearity=lasagne.nonlinearities.softmax)
l_diastole = CumSumLayer(l_d3b)
return {
"inputs":{
"sliced:data:ax": l0
},
"outputs":{
"systole": l_systole,
"diastole": l_diastole
}
}
def get_char2word(self, ic, avg=False):
suf = '_avg' if avg else ''
ec = L.EmbeddingLayer(
ic,
self.args.vc,
self.args.nc,
name='ec' + suf,
W=HeNormal() if not avg else Constant()) # (100, 24, 32, 16)
ec.params[ec.W].remove('regularizable')
if self.args.char_model == 'CNN':
lds = L.dimshuffle(ec, (0, 3, 1, 2)) # (100, 16, 24, 32)
ls = []
for n in self.args.ngrams:
lconv = L.Conv2DLayer(
lds,
self.args.nf, (1, n),
untie_biases=True,
W=HeNormal('relu') if not avg else Constant(),
name='conv_%d' % n + suf) # (100, 64/4, 24, 32-n+1)
lpool = L.MaxPool2DLayer(
lconv, (1, self.args.max_len - n + 1)) # (100, 64, 24, 1)
lpool = L.flatten(lpool, outdim=3) # (100, 16, 24)
lpool = L.dimshuffle(lpool, (0, 2, 1)) # (100, 24, 16)
ls.append(lpool)
xc = L.concat(ls, axis=2) # (100, 24, 64)
return xc
elif self.args.char_model == 'LSTM':
ml = L.ExpressionLayer(
ic, lambda x: T.neq(x, 0)) # mask layer (100, 24, 32)
ml = L.reshape(ml, (-1, self.args.max_len)) # (2400, 32)
gate_params = L.recurrent.Gate(W_in=Orthogonal(),
W_hid=Orthogonal())
cell_params = L.recurrent.Gate(W_in=Orthogonal(),
W_hid=Orthogonal(),
W_cell=None,
nonlinearity=tanh)
lstm_in = L.reshape(
ec, (-1, self.args.max_len, self.args.nc)) # (2400, 32, 16)
lstm_f = L.LSTMLayer(
lstm_in,
self.args.nw / 2,
mask_input=ml,
grad_clipping=10.,
learn_init=True,
peepholes=False,
precompute_input=True,
ingate=gate_params,
forgetgate=gate_params,
cell=cell_params,
outgate=gate_params,
# unroll_scan=True,
only_return_final=True,
name='forward' + suf) # (2400, 64)
lstm_b = L.LSTMLayer(
lstm_in,
self.args.nw / 2,
mask_input=ml,
grad_clipping=10.,
learn_init=True,
peepholes=False,
precompute_input=True,
ingate=gate_params,
forgetgate=gate_params,
cell=cell_params,
outgate=gate_params,
# unroll_scan=True,
only_return_final=True,
backwards=True,
name='backward' + suf) # (2400, 64)
remove_reg(lstm_f)
remove_reg(lstm_b)
if avg:
set_zero(lstm_f)
set_zero(lstm_b)
xc = L.concat([lstm_f, lstm_b], axis=1) # (2400, 128)
xc = L.reshape(xc,
(-1, self.args.sw, self.args.nw)) # (100, 24, 256)
return xc
def build_model(input_layer = None):
#################
# Regular model #
#################
l_4ch = nn.layers.InputLayer(data_sizes["sliced:data:chanzoom:4ch"])
l_2ch = nn.layers.InputLayer(data_sizes["sliced:data:chanzoom:2ch"])
#
l_4chr = reshape(l_4ch, (-1, 1, ) + l_4ch.output_shape[1:])
l_2chr = reshape(l_2ch, (-1, 1, ) + l_2ch.output_shape[1:])
# batch, features, timesteps, width, height
l_4ch_2a = deep_learning_layers.ConvolutionOver2DAxisLayer(l_4chr, num_filters=16, filter_size=(3, 3),
axis=(3,4), channel=1,
pad=(1,1),
W=nn.init.Orthogonal(),
b=nn.init.Constant(0.0),
)
l_4ch_2b = deep_learning_layers.ConvolutionOver2DAxisLayer(l_4ch_2a, num_filters=16, filter_size=(3, 3),
axis=(3,4), channel=1,
pad=(1,1),
W=nn.init.Orthogonal(),
b=nn.init.Constant(0.0),
)
l_4ch_2m = deep_learning_layers.MaxPoolOverAxisLayer(l_4ch_2b, pool_size=(2,), axis=(4,))
l_4ch_2r = nn.layers.DimshuffleLayer(l_4ch_2m, (0,2,3,1,4))
l_4ch_2r = reshape(l_4ch_2r, (-1, ) + l_4ch_2m.output_shape[1:2] + l_4ch_2m.output_shape[-1:])
l_4ch_d1 = nn.layers.DenseLayer(l_4ch_2r, num_units=64, W=nn.init.Orthogonal("relu"), b=nn.init.Constant(0.1), nonlinearity=nn.nonlinearities.rectify)
l_4ch_d2 = nn.layers.DenseLayer(l_4ch_d1, num_units=64, W=nn.init.Orthogonal("relu"), b=nn.init.Constant(0.1), nonlinearity=nn.nonlinearities.rectify)
l_4ch_mu = nn.layers.DenseLayer(l_4ch_d2, num_units=1, W=nn.init.Orthogonal("relu"), b=nn.init.Constant(0.1), nonlinearity=nn.nonlinearities.rectify)
l_4ch_sigma = nn.layers.DenseLayer(l_4ch_d2, num_units=1, W=nn.init.Orthogonal("relu"), b=nn.init.Constant(0.1), nonlinearity=nn.nonlinearities.rectify)
l_4ch_mu = reshape(l_4ch_mu, (-1, ) + l_4ch.output_shape[-3:-1])
l_4ch_sigma = reshape(l_4ch_sigma, (-1, ) + l_4ch.output_shape[-3:-1])
l_2ch_2a = deep_learning_layers.ConvolutionOver2DAxisLayer(l_2chr, num_filters=16, filter_size=(3, 3),
axis=(3,4), channel=1,
pad=(1,1),
W=nn.init.Orthogonal(),
b=nn.init.Constant(0.0),
)
l_2ch_2b = deep_learning_layers.ConvolutionOver2DAxisLayer(l_2ch_2a, num_filters=16, filter_size=(3, 3),
axis=(3,4), channel=1,
pad=(1,1),
W=nn.init.Orthogonal(),
b=nn.init.Constant(0.0),
)
l_2ch_2m = deep_learning_layers.MaxPoolOverAxisLayer(l_2ch_2b, pool_size=(2,), axis=(4,))
l_2ch_2r = nn.layers.DimshuffleLayer(l_2ch_2m, (0,2,3,1,4))
l_2ch_2r = reshape(l_2ch_2r, (-1, ) + l_2ch_2m.output_shape[1:2] + l_2ch_2m.output_shape[-1:])
l_2ch_d1 = nn.layers.DenseLayer(l_2ch_2r, num_units=64, W=nn.init.Orthogonal("relu"), b=nn.init.Constant(0.1), nonlinearity=nn.nonlinearities.rectify)
l_2ch_d2 = nn.layers.DenseLayer(l_2ch_d1, num_units=64, W=nn.init.Orthogonal("relu"), b=nn.init.Constant(0.1), nonlinearity=nn.nonlinearities.rectify)
l_2ch_mu = nn.layers.DenseLayer(l_2ch_d2, num_units=1, W=nn.init.Orthogonal("relu"), b=nn.init.Constant(0.1), nonlinearity=nn.nonlinearities.rectify)
l_2ch_sigma = nn.layers.DenseLayer(l_2ch_d2, num_units=1, W=nn.init.Orthogonal("relu"), b=nn.init.Constant(0.1), nonlinearity=nn.nonlinearities.rectify)
l_2ch_mu = reshape(l_2ch_mu, (-1, ) + l_2ch.output_shape[-3:-1])
l_2ch_sigma = reshape(l_2ch_sigma, (-1, ) + l_2ch.output_shape[-3:-1])
l_sys_and_dia = layers.IraLayer(l_4ch_mu, l_4ch_sigma, l_2ch_mu, l_2ch_sigma)
l_systole = nn.layers.SliceLayer(l_sys_and_dia, indices=0, axis=2)
l_diastole = nn.layers.SliceLayer(l_sys_and_dia, indices=1, axis=2)
return {
"inputs":{
"sliced:data:chanzoom:4ch": l_4ch,
"sliced:data:chanzoom:2ch": l_2ch,
},
"outputs": {
"systole": l_systole,
"diastole": l_diastole,
},
"regularizable": {
},
"meta_outputs": {
}
}
def build_model():
###############
# Sunny model #
###############
l0_sunny = InputLayer(data_sizes["sunny"])
l1a_sunny = ConvLayer(l0_sunny, num_filters=32, filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1b_sunny = WideConv2DDNNLayer(l1a_sunny, num_filters=32, filter_size=(3, 3), skip=2,
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1c_sunny = WideConv2DDNNLayer(l1b_sunny, num_filters=32, filter_size=(3, 3), skip=4,
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1d_sunny = WideConv2DDNNLayer(l1c_sunny, num_filters=32, filter_size=(3, 3), skip=8,
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1e_sunny = WideConv2DDNNLayer(l1d_sunny, num_filters=32, filter_size=(3, 3), skip=16,
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1f_sunny = WideConv2DDNNLayer(l1e_sunny, num_filters=1, filter_size=(3, 3), skip=32,
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
nonlinearity=lasagne.nonlinearities.sigmoid)
#l_sunny_segmentation = lasagne.layers.reshape(l1d_sunny, data_sizes["sunny"][:1] + l1d_sunny.output_shape[-2:])
l_sunny_segmentation = lasagne.layers.SliceLayer(l1f_sunny, indices=0, axis=1)
#################
# Regular model #
#################
l0 = InputLayer(data_sizes["sliced:data:ax"])
l0r = reshape(l0, (-1, 1, ) + data_sizes["sliced:data:ax"][-2:])
# first do the segmentation steps
l1a = ConvLayer(l0r, num_filters=32, filter_size=(3, 3),
pad='same',
W=l1a_sunny.W,
b=l1a_sunny.b)
l1b = WideConv2DDNNLayer(l1a, num_filters=32, filter_size=(3, 3), skip=2,
pad='same',
W=l1b_sunny.W,
b=l1b_sunny.b)
l1c = WideConv2DDNNLayer(l1b, num_filters=64, filter_size=(3, 3), skip=4,
pad='same',
W=l1c_sunny.W,
b=l1c_sunny.b)
l1d = WideConv2DDNNLayer(l1c, num_filters=64, filter_size=(3, 3), skip=8,
pad='same',
W=l1d_sunny.W,
b=l1d_sunny.b)
l1e = WideConv2DDNNLayer(l1d, num_filters=32, filter_size=(3, 3), skip=16,
pad='same',
W=l1e_sunny.W,
b=l1e_sunny.b)
l1f = WideConv2DDNNLayer(l1e, num_filters=1, filter_size=(3, 3), skip=32,
pad='same',
W=l1f_sunny.W,
b=l1f_sunny.b,
nonlinearity=lasagne.nonlinearities.sigmoid)
l_1r = reshape(l1f, data_sizes["sliced:data:ax"])
# returns (batch, time, 600) of probabilities
# TODO: it should also take into account resolution, etc.
volume_layer = GaussianApproximationVolumeLayer(l_1r)
# then use max and min over time for systole and diastole
l_systole = lasagne.layers.FlattenLayer(
lasagne.layers.FeaturePoolLayer(volume_layer,
pool_size=volume_layer.output_shape[1],
axis=1,
pool_function=T.min), outdim=2)
l_diastole = lasagne.layers.FlattenLayer(
lasagne.layers.FeaturePoolLayer(volume_layer,
pool_size=volume_layer.output_shape[1],
axis=1,
pool_function=T.max), outdim=2)
return {
"inputs":{
"sliced:data:ax": l0,
"sunny": l0_sunny,
},
"outputs":{
"systole": l_systole,
"diastole": l_diastole,
"segmentation": l_sunny_segmentation,
}
}
def dense_fused_convnets(self,
fusion_level,
fusion_type,
input_var1=None,
input_var2=None,
bottleneck_W=None,
weights_dir=None):
net = OrderedDict()
net['input_rgb'] = InputLayer((None, 4, 128, 128),
input_var=input_var1)
layer = 0
for i in range(self._net_specs_dict['num_conv_layers']):
# Add convolution layers
net['conv_rgb{0:d}'.format(i + 1)] = Conv2DLayer(
net.values()[layer],
num_filters=self._net_specs_dict['num_conv_filters'][i],
filter_size=(self._net_specs_dict['conv_filter_size'][i], ) *
2,
pad='same')
layer += 1
if self._net_specs_dict['num_conv_layers'] <= 2:
# Add pooling layers
net['pool_rgb{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
else:
if i < 4:
if (i + 1) % 2 == 0:
# Add pooling layers
net['pool_rgb{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
else:
if (i + 1) == 7:
# Add pooling layers
net['pool_rgb{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
# Fc-layers
net['fc1_rgb'] = DenseLayer(net.values()[layer],
self._net_specs_dict['num_fc_units'][0])
layer += 1
if fusion_level == 2:
# Add dropout layer
net['dropout1_rgb'] = dropout(net['fc1_rgb'],
p=self._model_hp_dict['p'])
layer += 1
net['fc2_rgb'] = DenseLayer(
net['dropout1_rgb'], self._net_specs_dict['num_fc_units'][1])
layer += 1
net['input_depth'] = InputLayer((None, 1, 128, 128),
input_var=input_var2)
layer += 1
for i in range(self._net_specs_dict['num_conv_layers']):
# Add convolution layers
net['conv_depth{0:d}'.format(i + 1)] = Conv2DLayer(
net.values()[layer],
num_filters=self._net_specs_dict['num_conv_filters'][i],
filter_size=(self._net_specs_dict['conv_filter_size'][i], ) *
2,
pad='same')
layer += 1
if self._net_specs_dict['num_conv_layers'] <= 2:
# Add pooling layers
net['pool_depth{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
else:
if i < 4:
if (i + 1) % 2 == 0:
# Add pooling layers
net['pool_depth{0:d}'.format(i+1)] =\
MaxPool2DLayer(net.values()[layer],
pool_size=(3, 3))
layer += 1
else:
if (i + 1) == 7:
# Add pooling layers
net['pool_depth{0:d}'.format(i+1)] =\
MaxPool2DLayer(net.values()[layer],
pool_size=(3, 3))
layer += 1
# Fc-layers
net['fc1_depth'] = DenseLayer(net.values()[layer],
self._net_specs_dict['num_fc_units'][0])
layer += 1
if fusion_level == 2:
# Add dropout layer
net['dropout1_depth'] = dropout(net['fc1_depth'],
p=self._model_hp_dict['p'])
layer += 1
net['fc2_depth'] = DenseLayer(
net['dropout1_depth'], self._net_specs_dict['num_fc_units'][1])
layer += 1
# Fuse ConvNets by fusion_level and fusion_type
if fusion_type == self.MAX:
net['merge'] =\
ElemwiseMergeLayer([net['fc%i_rgb' % fusion_level],
net['fc%i_depth' % fusion_level]],
T.maximum)
layer += 1
elif fusion_type == self.SUM:
net['merge'] =\
ElemwiseMergeLayer([net['fc%i_rgb' % fusion_level],
net['fc%i_depth' % fusion_level]],
T.add)
layer += 1
elif fusion_type == self.CONCAT:
net['merge'] = concat([
net['fc%i_rgb' % fusion_level],
net['fc%i_depth' % fusion_level]
])
layer += 1
elif fusion_type == self.CONCATCONV:
net['fc%i_rgb_res' % fusion_level] =\
reshape(net['fc%i_rgb' % fusion_level], ([0], 1, [1]))
layer += 1
net['fc%i_depth_res' % fusion_level] =\
reshape(net['fc%i_depth' % fusion_level], ([0], 1, [1]))
layer += 1
net['concat'] = concat([
net['fc%i_rgb_res' % fusion_level],
net['fc%i_depth_res' % fusion_level]
])
layer += 1
net['merge_con'] = Conv1DLayer(net['concat'],
num_filters=1,
filter_size=(1, ),
nonlinearity=None)
layer += 1
net['merge'] = reshape(net['merge_con'], ([0], [2]))
layer += 1
if fusion_level == 1:
# Add dropout layer
net['dropout1'] = dropout(net['merge'], p=self._model_hp_dict['p'])
layer += 1
net['fc2'] = DenseLayer(net['dropout1'],
self._net_specs_dict['num_fc_units'][1])
layer += 1
# Add dropout layer
net['dropout2'] = dropout(net['fc2'], p=self._model_hp_dict['p'])
layer += 1
else:
# Add dropout layer
net['dropout2'] = dropout(net['merge'], p=self._model_hp_dict['p'])
layer += 1
# Add output layer(linear activation because it's regression)
if bottleneck_W is not None:
# Add bottleneck layer
net['bottleneck'] = DenseLayer(net['dropout2'], 30)
# Add output layer(linear activation because it's regression)
net['output'] = DenseLayer(
net['bottleneck'],
3 * self._num_joints,
W=bottleneck_W[0:30],
nonlinearity=lasagne.nonlinearities.tanh)
else:
# Add output layer(linear activation because it's regression)
net['output'] = DenseLayer(
net['dropout2'],
3 * self._num_joints,
nonlinearity=lasagne.nonlinearities.tanh)
if weights_dir is not None:
lw = LoadWeights(weights_dir, net)
lw.load_weights_numpy()
return net
def __init__(
self,
input_shape,
output_dim,
hidden_sizes,
conv_filters,
conv_filter_sizes,
conv_strides,
conv_pads,
hidden_W_init=LI.GlorotUniform(),
hidden_b_init=LI.Constant(0.),
output_W_init=LI.GlorotUniform(),
output_b_init=LI.Constant(0.),
# conv_W_init=LI.GlorotUniform(), conv_b_init=LI.Constant(0.),
hidden_nonlinearity=LN.rectify,
output_nonlinearity=LN.softmax,
name=None,
input_var=None):
"""
Network of convolution layers followd by dense layers.
:param input_shape: input shape, i.e. for 3D images it should be (channels, rows, cols)
:type input_shape: tuple
:param output_dim: number of output units/neurons
:type output_dim: int
:param hidden_sizes: list of unit numbers
:type hidden_sizes: int-tuple
:param conv_filters: number of convolution filters (in literature depth of conv layer)
:type conv_filters: tuple
:param conv_filter_sizes:
:type conv_filter_sizes: tuple
:param conv_strides:
:param conv_pads:
:param hidden_W_init:
:param hidden_b_init:
:param output_W_init:
:param output_b_init:
:param hidden_nonlinearity:
:param output_nonlinearity:
:param name:
:param input_var:
Note
----
The trick here is that it flattens the input.
"""
if name is None:
prefix = ""
else:
prefix = name + "_"
if len(input_shape) == 3:
l_in = L.InputLayer(shape=(None, np.prod(input_shape)),
input_var=input_var)
l_hid = L.reshape(l_in, ([0], ) + input_shape)
elif len(input_shape) == 2:
l_in = L.InputLayer(shape=(None, np.prod(input_shape)),
input_var=input_var)
input_shape = (1, ) + input_shape
l_hid = L.reshape(l_in, ([0], ) + input_shape)
else:
l_in = L.InputLayer(shape=(None, ) + input_shape,
input_var=input_var)
l_hid = l_in
for idx, conv_filter, filter_size, stride, pad in zip(
range(len(conv_filters)),
conv_filters,
conv_filter_sizes,
conv_strides,
conv_pads,
):
l_hid = L.Conv2DLayer(
l_hid,
num_filters=conv_filter,
filter_size=filter_size,
stride=(stride, stride),
pad=pad,
nonlinearity=hidden_nonlinearity,
name="%sconv_hidden_%d" % (prefix, idx),
W=hidden_W_init,
b=hidden_b_init,
)
for idx, hidden_size in enumerate(hidden_sizes):
l_hid = L.DenseLayer(
l_hid,
num_units=hidden_size,
nonlinearity=hidden_nonlinearity,
name="%shidden_%d" % (prefix, idx),
W=hidden_W_init,
b=hidden_b_init,
)
l_out = L.DenseLayer(
l_hid,
num_units=output_dim,
nonlinearity=output_nonlinearity,
name="%soutput" % (prefix, ),
W=output_W_init,
b=output_b_init,
)
self._l_in = l_in
self._l_out = l_out
self._input_var = l_in.input_var
def build_net(nz=200):
gen = InputLayer(shape=(None, nz))
gen = DenseLayer(incoming=gen, num_units=1024 * 4 * 4)
gen = reshape(incoming=gen, shape=(-1, 1024, 4, 4))
gen = batch_norm(
Deconv2DLayer(incoming=gen,
num_filters=512,
filter_size=4,
stride=2,
nonlinearity=relu,
crop=1))
gen = batch_norm(
Deconv2DLayer(incoming=gen,
num_filters=256,
filter_size=4,
stride=2,
nonlinearity=relu,
crop=1))
gen = batch_norm(
Deconv2DLayer(incoming=gen,
num_filters=128,
filter_size=4,
stride=2,
nonlinearity=relu,
crop=1))
gen = Deconv2DLayer(incoming=gen,
num_filters=1,
filter_size=4,
stride=2,
nonlinearity=sigmoid,
crop=1) ### or tanh?
dis = InputLayer(shape=(None, 1, 64, 64))
dis = batch_norm(
Conv2DLayer(incoming=dis,
num_filters=128,
filter_size=5,
stride=2,
nonlinearity=lrelu(0.2),
pad=2))
dis = batch_norm(
Conv2DLayer(incoming=dis,
num_filters=256,
filter_size=5,
stride=2,
nonlinearity=lrelu(0.2),
pad=2))
dis = batch_norm(
Conv2DLayer(incoming=dis,
num_filters=512,
filter_size=5,
stride=2,
nonlinearity=lrelu(0.2),
pad=2))
dis = batch_norm(
Conv2DLayer(incoming=dis,
num_filters=1024,
filter_size=5,
stride=2,
nonlinearity=lrelu(0.2),
pad=2))
dis = reshape(incoming=dis, shape=(-1, 1024 * 4 * 4))
dis = DenseLayer(incoming=dis, num_units=1, nonlinearity=None)
return gen, dis
def __init__(
self,
input_shape,
output_dim,
hidden_sizes,
conv_filters,
conv_filter_sizes,
conv_strides,
conv_pads,
hidden_W_init=LI.GlorotUniform(),
hidden_b_init=LI.Constant(0.),
output_W_init=LI.GlorotUniform(),
output_b_init=LI.Constant(0.),
# conv_W_init=LI.GlorotUniform(), conv_b_init=LI.Constant(0.),
hidden_nonlinearity=LN.rectify,
output_nonlinearity=LN.softmax,
name=None,
input_var=None):
if name is None:
prefix = ""
else:
prefix = name + "_"
if len(input_shape) == 3:
l_in = L.InputLayer(shape=(None, np.prod(input_shape)),
input_var=input_var)
l_hid = L.reshape(l_in, ([0], ) + input_shape)
elif len(input_shape) == 2:
l_in = L.InputLayer(shape=(None, np.prod(input_shape)),
input_var=input_var)
input_shape = (1, ) + input_shape
l_hid = L.reshape(l_in, ([0], ) + input_shape)
else:
l_in = L.InputLayer(shape=(None, ) + input_shape,
input_var=input_var)
l_hid = l_in
for idx, conv_filter, filter_size, stride, pad in zip(
range(len(conv_filters)),
conv_filters,
conv_filter_sizes,
conv_strides,
conv_pads,
):
l_hid = L.Conv2DLayer(
l_hid,
num_filters=conv_filter,
filter_size=filter_size,
stride=(stride, stride),
pad=pad,
nonlinearity=hidden_nonlinearity,
name="%sconv_hidden_%d" % (prefix, idx),
convolution=wrapped_conv,
)
conv_out = l_hid
for idx, hidden_size in enumerate(hidden_sizes):
l_hid = L.DenseLayer(
l_hid,
num_units=hidden_size,
nonlinearity=hidden_nonlinearity,
name="%shidden_%d" % (prefix, idx),
W=hidden_W_init,
b=hidden_b_init,
)
l_out = L.DenseLayer(
l_hid,
num_units=output_dim,
nonlinearity=output_nonlinearity,
name="%soutput" % (prefix, ),
W=output_W_init,
b=output_b_init,
)
self._l_in = l_in
self._l_out = l_out
self._input_var = l_in.input_var
self._conv_out = conv_out
def build_model():
#################
# Regular model #
#################
data_size = data_sizes["sliced:data:ax:noswitch"]
l0 = InputLayer(data_size)
l0r = batch_norm(reshape(l0, (
-1,
1,
) + data_size[1:]))
# (batch, channel, axis, time, x, y)
# convolve over time
l1 = batch_norm(
ConvolutionOverAxisLayer(
l0r,
num_filters=4,
filter_size=(3, ),
axis=(3, ),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.0),
))
l1m = batch_norm(MaxPoolOverAxisLayer(l1, pool_size=(4, ), axis=(3, )))
# convolve over x and y
l2a = batch_norm(
ConvolutionOver2DAxisLayer(
l1m,
num_filters=8,
filter_size=(3, 3),
axis=(4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.0),
))
l2b = batch_norm(
ConvolutionOver2DAxisLayer(
l2a,
num_filters=8,
filter_size=(3, 3),
axis=(4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.0),
))
l2m = batch_norm(MaxPoolOver2DAxisLayer(l2b, pool_size=(2, 2),
axis=(4, 5)))
# convolve over x, y, time
l3a = batch_norm(
ConvolutionOver3DAxisLayer(
l2m,
num_filters=64,
filter_size=(3, 3, 3),
axis=(3, 4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
))
l3b = batch_norm(
ConvolutionOver2DAxisLayer(
l3a,
num_filters=64,
filter_size=(3, 3),
axis=(4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
))
l3m = batch_norm(MaxPoolOver2DAxisLayer(l3b, pool_size=(2, 2),
axis=(4, 5)))
# convolve over time
l4 = batch_norm(
ConvolutionOverAxisLayer(
l3m,
num_filters=64,
filter_size=(3, ),
axis=(3, ),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
))
l4m = batch_norm(MaxPoolOverAxisLayer(l4, pool_size=(2, ), axis=(2, )))
# maxpool over axis
l5 = batch_norm(MaxPoolOverAxisLayer(l3m, pool_size=(4, ), axis=(2, )))
# convolve over x and y
l6a = batch_norm(
ConvolutionOver2DAxisLayer(
l5,
num_filters=128,
filter_size=(3, 3),
axis=(4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
))
l6b = batch_norm(
ConvolutionOver2DAxisLayer(
l6a,
num_filters=128,
filter_size=(3, 3),
axis=(4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
))
l6m = batch_norm(MaxPoolOver2DAxisLayer(l6b, pool_size=(2, 2),
axis=(4, 5)))
# convolve over time and x,y
l7 = ConvolutionOver3DAxisLayer(
l6m,
num_filters=128,
filter_size=(3, 3, 3),
axis=(3, 4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l8 = lasagne.layers.DropoutLayer(l7, p=0.5)
l_d3a = lasagne.layers.DenseLayer(l8, num_units=128)
l_d3b = lasagne.layers.DropoutLayer(l_d3a)
l_systole = CumSumLayer(
lasagne.layers.DenseLayer(l_d3b,
num_units=600,
nonlinearity=lasagne.nonlinearities.softmax))
l_d3c = lasagne.layers.DenseLayer(l8, num_units=128)
l_d3d = lasagne.layers.DropoutLayer(l_d3c)
l_diastole = CumSumLayer(
lasagne.layers.DenseLayer(l_d3d,
num_units=600,
nonlinearity=lasagne.nonlinearities.softmax))
return {
"inputs": {
"sliced:data:ax:noswitch": l0
},
"outputs": {
"systole": l_systole,
"diastole": l_diastole,
},
"regularizable": {
l_d3a: 0.25,
l_d3c: 0.25,
l_systole: 0.25,
l_diastole: 0.25,
}
}
def build_model():
#################
# Regular model #
#################
l0 = InputLayer(data_sizes["sliced:data:ax"])
l0r = reshape(l0, (-1, 1, ) + data_sizes["sliced:data:ax"][1:])
# (batch, channel, time, axis, x, y)
# convolve over time with small filter
l0t = ConvolutionOverAxisLayer(l0r, num_filters=2, filter_size=(5,), stride=(3,), axis=2, channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l0r = reshape(l0t, (-1, 1, ) + data_sizes["sliced:data:ax"][-2:])
# first do the segmentation steps
l1a = ConvLayer(l0r, num_filters=32, filter_size=(5, 5), stride=2,
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1b = ConvLayer(l1a, num_filters=32, filter_size=(5, 5), stride=2,
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1b_m = MaxPoolLayer(l1b, pool_size=(2,2))
l1c = ConvLayer(l1b_m, num_filters=64, filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1f = ConvLayer(l1c, num_filters=32, filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1))
l1f_m = MaxPoolLayer(l1f, pool_size=(2,2))
l1f_r = reshape(l1f_m, (batch_size, 2, 9, 15, 32, 4, 4))
l0t = ConvolutionOverAxisLayer(l1f_r, num_filters=32, filter_size=(3,), stride=(1,),
axis=3, channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l_d3 = lasagne.layers.DenseLayer(l0t,
num_units=2,
nonlinearity=lasagne.nonlinearities.identity)
l_systole = MuLogSigmaErfLayer(l_d3)
l_d3b = lasagne.layers.DenseLayer(l0t,
num_units=2,
nonlinearity=lasagne.nonlinearities.identity)
l_diastole = MuLogSigmaErfLayer(l_d3b)
return {
"inputs":{
"sliced:data:ax": l0
},
"outputs":{
"systole": l_systole,
"diastole": l_diastole
}
}
def build_model():
'''
Adapted from https://github.com/Lasagne/Recipes/tree/master/papers/deep_residual_learning.
Tweaked to be consistent with 'Identity Mappings in Deep Residual Networks', Kaiming He et al. 2016 (https://arxiv.org/abs/1603.05027)
And 'Wide Residual Networks', Sergey Zagoruyko, Nikos Komodakis 2016 (http://arxiv.org/pdf/1605.07146v1.pdf)
Depth = 6n + 2
'''
n = 6
k = 4
n_filters = {0: 16, 1: 16 * k, 2: 32 * k, 3: 64 * k}
l_in = InputLayer(shape=(None, ) + nn_input_shape)
l = DimshuffleLayer(l_in, pattern=(0, 'x', 1, 2, 3))
# first layer, output is 16 x 64 x 64
l = batch_norm(
ConvLayer(l,
num_filters=n_filters[0],
filter_size=3,
stride=1,
nonlinearity=rectify,
pad='same',
W=he_norm))
# first stack of residual blocks, output is 32 x 64 x 64
l = residual_block(l, first=True, filters=n_filters[1])
for _ in range(1, n):
l = residual_block(l, filters=n_filters[1])
# second stack of residual blocks, output is 64 x 32 x 32
l = residual_block(l, increase_dim=True, filters=n_filters[2])
for _ in range(1, (n + 2)):
l = residual_block(l, filters=n_filters[2])
# third stack of residual blocks, output is 128 x 16 x 16
l = residual_block(l, increase_dim=True, filters=n_filters[3])
for _ in range(1, (n + 2)):
l = residual_block(l, filters=n_filters[3])
bn_post_conv = BatchNormLayer(l)
bn_post_relu = NonlinearityLayer(bn_post_conv, rectify)
# average pooling
avg_pool = GlobalPoolLayer(bn_post_relu)
# average pooling
avg_pool = GlobalPoolLayer(bn_post_relu)
sigm = DenseLayer(avg_pool,
num_units=1,
W=lasagne.init.Constant(0.0),
b=None,
nonlinearity=lasagne.nonlinearities.sigmoid)
l_out = reshape(sigm, shape=(-1, ))
return {
"inputs": {
"bcolzall:3d": l_in,
},
"outputs": {
"predicted_probability": l_out
},
}
def build_model():
#################
# Regular model #
#################
input_key = "sliced:data:ax:noswitch"
data_size = data_sizes[input_key]
l0 = InputLayer(data_size)
l0r = batch_norm(reshape(l0, (-1, 1, ) + data_size[1:]))
# (batch, channel, axis, time, x, y)
# convolve over time
l1 = batch_norm(ConvolutionOverAxisLayer(l0r, num_filters=8, filter_size=(3,), axis=(3,), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.0),
))
l1m = batch_norm(MaxPoolOverAxisLayer(l1, pool_size=(4,), axis=(3,)))
# convolve over x and y
l2a = batch_norm(ConvolutionOver2DAxisLayer(l1m, num_filters=8, filter_size=(3, 3),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.0),
))
l2b = batch_norm(ConvolutionOver2DAxisLayer(l2a, num_filters=8, filter_size=(3, 3),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.0),
))
l2m = batch_norm(MaxPoolOver2DAxisLayer(l2b, pool_size=(2, 2), axis=(4,5)))
# convolve over x, y, time
l3a = batch_norm(ConvolutionOver3DAxisLayer(l2m, num_filters=32, filter_size=(3, 3, 3),
axis=(3,4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
))
l3b = batch_norm(ConvolutionOver2DAxisLayer(l3a, num_filters=32, filter_size=(3, 3),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
))
l3m = batch_norm(MaxPoolOver2DAxisLayer(l3b, pool_size=(2, 2), axis=(4,5)))
# convolve over time
l4 = batch_norm(ConvolutionOverAxisLayer(l3m, num_filters=32, filter_size=(3,), axis=(3,), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
))
l4m = batch_norm(MaxPoolOverAxisLayer(l4, pool_size=(2,), axis=(2,)))
# maxpool over axis
l5 = batch_norm(MaxPoolOverAxisLayer(l3m, pool_size=(4,), axis=(2,)))
# convolve over x and y
l6a = batch_norm(ConvolutionOver2DAxisLayer(l5, num_filters=128, filter_size=(3, 3),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
))
l6b = batch_norm(ConvolutionOver2DAxisLayer(l6a, num_filters=128, filter_size=(3, 3),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
))
l6m = batch_norm(MaxPoolOver2DAxisLayer(l6b, pool_size=(2, 2), axis=(4,5)))
# convolve over time and x,y, is sparse reduction layer
l7 = ConvolutionOver3DAxisLayer(l6m, num_filters=32, filter_size=(3,3,3), axis=(3,4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
key_scale = "area_per_pixel:sax"
l_scale = InputLayer(data_sizes[key_scale])
# Systole Dense layers
ldsys1 = lasagne.layers.DenseLayer(l7, num_units=512,
W=lasagne.init.Orthogonal("relu"),
b=lasagne.init.Constant(0.1),
nonlinearity=lasagne.nonlinearities.rectify)
ldsys1drop = lasagne.layers.dropout(ldsys1, p=0.5)
ldsys2 = lasagne.layers.DenseLayer(ldsys1drop, num_units=128,
W=lasagne.init.Orthogonal("relu"),
b=lasagne.init.Constant(0.1),
nonlinearity=lasagne.nonlinearities.rectify)
ldsys2drop = lasagne.layers.dropout(ldsys2, p=0.5)
ldsys3 = lasagne.layers.DenseLayer(ldsys2drop, num_units=1,
b=lasagne.init.Constant(0.1),
nonlinearity=lasagne.nonlinearities.identity)
l_systole = layers.MuConstantSigmaErfLayer(layers.ScaleLayer(ldsys3, scale=l_scale), sigma=0.0)
# Diastole Dense layers
lddia1 = lasagne.layers.DenseLayer(l7, num_units=512,
W=lasagne.init.Orthogonal("relu"),
b=lasagne.init.Constant(0.1),
nonlinearity=lasagne.nonlinearities.rectify)
lddia1drop = lasagne.layers.dropout(lddia1, p=0.5)
lddia2 = lasagne.layers.DenseLayer(lddia1drop, num_units=128,
W=lasagne.init.Orthogonal("relu"),
b=lasagne.init.Constant(0.1),
nonlinearity=lasagne.nonlinearities.rectify)
lddia2drop = lasagne.layers.dropout(lddia2, p=0.5)
lddia3 = lasagne.layers.DenseLayer(lddia2drop, num_units=1,
b=lasagne.init.Constant(0.1),
nonlinearity=lasagne.nonlinearities.identity)
l_diastole = layers.MuConstantSigmaErfLayer(layers.ScaleLayer(lddia3, scale=l_scale), sigma=0.0)
return {
"inputs":{
input_key: l0,
key_scale: l_scale,
},
"outputs": {
"systole": l_systole,
"diastole": l_diastole,
},
"regularizable": {
ldsys1: l2_weight,
ldsys2: l2_weight,
ldsys3: l2_weight,
lddia1: l2_weight,
lddia2: l2_weight,
lddia3: l2_weight,
},
}
def _get_l_out(self, input_vars, multi_utt=None):
check_options(self.options)
id_tag = (self.id + '/') if self.id else ''
input_var = input_vars[0]
extra_vars = input_vars[1:]
if multi_utt is None:
l_in = InputLayer(shape=(None, self.seq_vec.max_len), input_var=input_var,
name=id_tag + 'desc_input')
l_in_flattened = l_in
else:
l_in = InputLayer(shape=(None, multi_utt, self.seq_vec.max_len), input_var=input_var,
name=id_tag + 'desc_input')
l_in_flattened = reshape(l_in, (-1, self.seq_vec.max_len),
name=id_tag + 'input_flattened')
l_in_embed, context_vars = self.get_embedding_layer(l_in_flattened, extra_vars)
cell = CELLS[self.options.listener_cell]
cell_kwargs = {
'grad_clipping': self.options.listener_grad_clipping,
'num_units': self.options.listener_cell_size,
}
if self.options.listener_cell == 'LSTM':
cell_kwargs['forgetgate'] = Gate(b=Constant(self.options.listener_forget_bias))
if self.options.listener_cell != 'GRU':
cell_kwargs['nonlinearity'] = NONLINEARITIES[self.options.listener_nonlinearity]
l_rec1 = cell(l_in_embed, name=id_tag + 'rec1', only_return_final=True, **cell_kwargs)
if self.options.listener_bidi:
l_rec1_backwards = cell(l_in_embed, name=id_tag + 'rec1_back', backwards=True,
only_return_final=True, **cell_kwargs)
l_rec1 = ConcatLayer([l_rec1, l_rec1_backwards], axis=1,
name=id_tag + 'rec1_bidi_concat')
if self.options.listener_dropout > 0.0:
l_rec1_drop = DropoutLayer(l_rec1, p=self.options.listener_dropout,
name=id_tag + 'rec1_drop')
else:
l_rec1_drop = l_rec1
# (batch_size [ * multi_utt], repr_size)
l_pred_mean = DenseLayer(l_rec1_drop, num_units=self.color_vec.output_size,
nonlinearity=None, name=id_tag + 'pred_mean')
# (batch_size [ * multi_utt], repr_size * repr_size)
l_pred_covar_vec = DenseLayer(l_rec1_drop, num_units=self.color_vec.output_size ** 2,
# initially produce identity matrix
b=np.eye(self.color_vec.output_size,
dtype=theano.config.floatX).ravel(),
nonlinearity=None, name=id_tag + 'pred_covar_vec')
# (batch_size [ * multi_utt], repr_size, repr_size)
l_pred_covar = reshape(l_pred_covar_vec, ([0], self.color_vec.output_size,
self.color_vec.output_size),
name=id_tag + 'pred_covar')
if multi_utt is not None:
l_pred_mean = reshape(l_pred_mean, (-1, multi_utt, self.color_vec.output_size),
name=id_tag + 'pred_mean_reshape')
l_pred_covar = reshape(l_pred_covar, (-1, multi_utt, self.color_vec.output_size,
self.color_vec.output_size),
name=id_tag + 'pred_covar_reshape')
# Context repr has shape (batch_size, context_len * repr_size)
l_context_repr, context_inputs = self.color_vec.get_input_layer(
context_vars,
cell_size=self.options.listener_cell_size,
context_len=self.context_len,
id=self.id
)
l_context_points = reshape(l_context_repr, ([0], self.context_len,
self.color_vec.output_size))
# (batch_size, [multi_utt,] context_len)
l_unnorm_scores = GaussianScoreLayer(l_context_points, l_pred_mean, l_pred_covar,
name=id_tag + 'gaussian_score')
if multi_utt is not None:
l_unnorm_scores = reshape(l_unnorm_scores, (-1, self.context_len),
name=id_tag + 'gaussian_score_reshape')
# (batch_size [ * multi_utt], context_len)
# XXX: returning probs for normal models, log probs for AC model!
# This is really surprising and definitely not the best solution.
# We should be using log probs everywhere for stability...
final_softmax = (softmax if multi_utt is None else logit_softmax_nd(axis=2))
l_scores = NonlinearityLayer(l_unnorm_scores, nonlinearity=final_softmax,
name=id_tag + 'scores')
if multi_utt is not None:
l_scores = reshape(l_unnorm_scores, (-1, multi_utt, self.context_len),
name=id_tag + 'scores_reshape')
self.gaussian_fn = theano.function(input_vars, [get_output(l_pred_mean,
deterministic=True),
get_output(l_pred_covar,
deterministic=True),
get_output(l_context_points,
deterministic=True),
get_output(l_unnorm_scores,
deterministic=True)],
name=id_tag + 'gaussian',
on_unused_input='ignore')
self.repr_fn = theano.function(input_vars, get_output(l_rec1_drop,
deterministic=True),
name=id_tag + 'repr',
on_unused_input='ignore')
return l_scores, [l_in] + context_inputs
def build_model():
###############
# Sunny model #
###############
l0_sunny = InputLayer(data_sizes["sunny"])
sunny_layers = [l0_sunny]
for i in xrange(1,21):
layer = ConvLayer(sunny_layers[-1], num_filters=8, filter_size=((1, 7) if i%2==0 else (7, 1)),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
sunny_layers.append(layer)
l1_sunny = ConvLayer(sunny_layers[-1], num_filters=1, filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
nonlinearity=lasagne.nonlinearities.sigmoid)
#l_sunny_segmentation = lasagne.layers.reshape(l1d_sunny, data_sizes["sunny"][:1] + l1d_sunny.output_shape[-2:])
l_sunny_segmentation = lasagne.layers.SliceLayer(l1_sunny, indices=0, axis=1)
#################
# Regular model #
#################
l0 = InputLayer(data_sizes["sliced:data:ax"])
l0r = reshape(l0, (-1, 1, ) + data_sizes["sliced:data:ax"][-2:])
# first do the segmentation steps
layers = [l0r]
for i in xrange(1,21):
layer = ConvLayer(layers[-1], num_filters=8, filter_size=((1, 7) if i%2==0 else (7, 1)),
pad='same',
W=sunny_layers[i].W,
b=sunny_layers[i].b,
)
layers.append(layer)
l1f = ConvLayer(layers[-1], num_filters=1, filter_size=(3, 3),
pad='same',
W=l1_sunny.W,
b=l1_sunny.b,
nonlinearity=lasagne.nonlinearities.sigmoid)
l_1r = reshape(l1f, data_sizes["sliced:data:ax"])
# returns (batch, time, 600) of probabilities
# TODO: it should also take into account resolution, etc.
volume_layer = GaussianApproximationVolumeLayer(l_1r)
# then use max and min over time for systole and diastole
l_systole = lasagne.layers.FlattenLayer(
lasagne.layers.FeaturePoolLayer(volume_layer,
pool_size=volume_layer.output_shape[1],
axis=1,
pool_function=T.min), outdim=2)
l_diastole = lasagne.layers.FlattenLayer(
lasagne.layers.FeaturePoolLayer(volume_layer,
pool_size=volume_layer.output_shape[1],
axis=1,
pool_function=T.max), outdim=2)
return {
"inputs":{
"sliced:data:ax": l0,
"sunny": l0_sunny,
},
"outputs":{
"systole": l_systole,
"diastole": l_diastole,
"segmentation": l_sunny_segmentation,
}
}
def build_model():
#################
# Regular model #
#################
l0 = InputLayer(data_sizes["sliced:data:ax"])
l0r = reshape(l0, (-1, 1, ) + data_sizes["sliced:data:ax"][1:])
# (batch, channel, time, axis, x, y)
# convolve over time
l1 = ConvolutionOverAxisLayer(l0r, num_filters=4, filter_size=(3,), axis=(2,), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1b = ConvolutionOverAxisLayer(l1, num_filters=4, filter_size=(3,), axis=(2,), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1m = MaxPoolOverAxisLayer(l1b, pool_size=(2,), axis=(2,))
# convolve over x and y
l2a = ConvolutionOver2DAxisLayer(l1m, num_filters=32, filter_size=(3, 3),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l2b = ConvolutionOver2DAxisLayer(l2a, num_filters=32, filter_size=(3, 3),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l2m = MaxPoolOver2DAxisLayer(l2b, pool_size=(2, 2), axis=(4,5))
# convolve over x, y and axis
l3a = ConvolutionOver3DAxisLayer(l2m, num_filters=64, filter_size=(3, 3, 3),
axis=(3,4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l3b = ConvolutionOver3DAxisLayer(l3a, num_filters=64, filter_size=(3, 3, 3),
axis=(3,4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l3c = ConvolutionOver3DAxisLayer(l3b, num_filters=64, filter_size=(3, 3, 3),
axis=(3,4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l3m = MaxPoolOver3DAxisLayer(l3c, pool_size=(2, 2, 2), axis=(3,4,5))
# convolve over time
l4 = ConvolutionOverAxisLayer(l3m, num_filters=64, filter_size=(3,), axis=(2,), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l4m = MaxPoolOverAxisLayer(l4, pool_size=(2,), axis=(2,))
# convolve over axis
l5 = ConvolutionOverAxisLayer(l4m, num_filters=128, filter_size=(3,), axis=(3,), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
# convolve over x and y
l6a = ConvolutionOver2DAxisLayer(l5, num_filters=128, filter_size=(3, 3),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l6b = ConvolutionOver2DAxisLayer(l6a, num_filters=128, filter_size=(3, 3),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l6m = MaxPoolOver2DAxisLayer(l6b, pool_size=(2, 2), axis=(4,5))
# convolve over time and x,y
l7 = ConvolutionOver3DAxisLayer(l6m, num_filters=64, filter_size=(3,3,3), axis=(2,4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l8 = lasagne.layers.DropoutLayer(l7)
l_d3a = lasagne.layers.DenseLayer(l8,
num_units=100)
l_d3b = lasagne.layers.DropoutLayer(l_d3a)
l_systole = lasagne.layers.DenseLayer(l_d3b,
num_units=600,
nonlinearity=lasagne.nonlinearities.softmax)
l_diastole = lasagne.layers.DenseLayer(l_d3b,
num_units=600,
nonlinearity=lasagne.nonlinearities.softmax)
return {
"inputs":{
"sliced:data:ax": l0
},
"outputs": {
"systole:onehot": l_systole,
"diastole:onehot": l_diastole,
}
}
def build_model():
#################
# Regular model #
#################
l0 = InputLayer(data_sizes["sliced:data:ax"])
l0r = reshape(l0, (
-1,
1,
) + data_sizes["sliced:data:ax"][1:])
# (batch, channel, time, axis, x, y)
# convolve over time
l1 = ConvolutionOverAxisLayer(
l0r,
num_filters=4,
filter_size=(3, ),
axis=(2, ),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1b = ConvolutionOverAxisLayer(
l1,
num_filters=4,
filter_size=(3, ),
axis=(2, ),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1m = MaxPoolOverAxisLayer(l1b, pool_size=(2, ), axis=(2, ))
# convolve over x and y
l2a = ConvolutionOver2DAxisLayer(
l1m,
num_filters=32,
filter_size=(3, 3),
axis=(4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l2b = ConvolutionOver2DAxisLayer(
l2a,
num_filters=32,
filter_size=(3, 3),
axis=(4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l2m = MaxPoolOver2DAxisLayer(l2b, pool_size=(2, 2), axis=(4, 5))
# convolve over x, y and axis
l3a = ConvolutionOver3DAxisLayer(
l2m,
num_filters=64,
filter_size=(3, 3, 3),
axis=(3, 4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l3b = ConvolutionOver3DAxisLayer(
l3a,
num_filters=64,
filter_size=(3, 3, 3),
axis=(3, 4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l3c = ConvolutionOver3DAxisLayer(
l3b,
num_filters=64,
filter_size=(3, 3, 3),
axis=(3, 4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l3m = MaxPoolOver3DAxisLayer(l3c, pool_size=(2, 2, 2), axis=(3, 4, 5))
# convolve over time
l4 = ConvolutionOverAxisLayer(
l3m,
num_filters=64,
filter_size=(3, ),
axis=(2, ),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l4m = MaxPoolOverAxisLayer(l4, pool_size=(2, ), axis=(2, ))
# convolve over axis
l5 = ConvolutionOverAxisLayer(
l4m,
num_filters=128,
filter_size=(3, ),
axis=(3, ),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
# convolve over x and y
l6a = ConvolutionOver2DAxisLayer(
l5,
num_filters=128,
filter_size=(3, 3),
axis=(4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l6b = ConvolutionOver2DAxisLayer(
l6a,
num_filters=128,
filter_size=(3, 3),
axis=(4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l6m = MaxPoolOver2DAxisLayer(l6b, pool_size=(2, 2), axis=(4, 5))
# convolve over time and x,y
l7 = ConvolutionOver3DAxisLayer(
l6m,
num_filters=32,
filter_size=(3, 3, 3),
axis=(2, 4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l8 = lasagne.layers.DropoutLayer(l7)
l_d3a = lasagne.layers.DenseLayer(l8, num_units=600)
l_d3b = lasagne.layers.DropoutLayer(l_d3a)
l_systole = lasagne.layers.DenseLayer(
l_d3b, num_units=600, nonlinearity=lasagne.nonlinearities.softmax)
l_d3c = lasagne.layers.DenseLayer(l8, num_units=600)
l_d3d = lasagne.layers.DropoutLayer(l_d3c)
l_diastole = lasagne.layers.DenseLayer(
l_d3d, num_units=600, nonlinearity=lasagne.nonlinearities.softmax)
return {
"inputs": {
"sliced:data:ax": l0
},
"outputs": {
"systole:onehot": l_systole,
"diastole:onehot": l_diastole,
},
"regularizable": {
l_d3a: 0.25,
l_d3c: 0.25,
l_systole: 0.25,
l_diastole: 0.25,
}
}
def build_model():
#################
# Regular model #
#################
l0 = InputLayer(data_sizes["sliced:data:ax"])
l0r = batch_norm(reshape(l0, (-1, 1) + data_sizes["sliced:data:ax"][1:]))
# (batch, channel, time, axis, x, y)
# convolve over time
l1 = batch_norm(
ConvolutionOverAxisLayer(
l0r,
num_filters=16,
filter_size=(3,),
axis=(2,),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.0),
)
)
l1b = batch_norm(
ConvolutionOverAxisLayer(
l1,
num_filters=16,
filter_size=(3,),
axis=(2,),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.0),
)
)
l1m = batch_norm(MaxPoolOverAxisLayer(l1b, pool_size=(4,), axis=(2,)))
# convolve over x and y
l2a = batch_norm(
ConvolutionOver2DAxisLayer(
l1m,
num_filters=32,
filter_size=(3, 3),
axis=(4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.0),
)
)
l2b = batch_norm(
ConvolutionOver2DAxisLayer(
l2a,
num_filters=32,
filter_size=(3, 3),
axis=(4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.0),
)
)
l2m = batch_norm(MaxPoolOver2DAxisLayer(l2b, pool_size=(2, 2), axis=(4, 5)))
# convolve over x, y, time
l3a = batch_norm(
ConvolutionOver3DAxisLayer(
l2m,
num_filters=64,
filter_size=(3, 3, 3),
axis=(2, 4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
)
l3b = batch_norm(
ConvolutionOver2DAxisLayer(
l3a,
num_filters=64,
filter_size=(3, 3),
axis=(4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
)
l3m = batch_norm(MaxPoolOver2DAxisLayer(l3b, pool_size=(2, 2), axis=(4, 5)))
# convolve over time
l4 = batch_norm(
ConvolutionOverAxisLayer(
l3m,
num_filters=64,
filter_size=(3,),
axis=(2,),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
)
l4m = batch_norm(MaxPoolOverAxisLayer(l4, pool_size=(2,), axis=(2,)))
# maxpool over axis
l5 = batch_norm(MaxPoolOverAxisLayer(l3m, pool_size=(4,), axis=(3,)))
# convolve over x and y
l6a = batch_norm(
ConvolutionOver2DAxisLayer(
l5,
num_filters=128,
filter_size=(3, 3),
axis=(4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
)
l6b = batch_norm(
ConvolutionOver2DAxisLayer(
l6a,
num_filters=128,
filter_size=(3, 3),
axis=(4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
)
l6m = batch_norm(MaxPoolOver2DAxisLayer(l6b, pool_size=(2, 2), axis=(4, 5)))
# convolve over time and x,y
l7 = ConvolutionOver3DAxisLayer(
l6m,
num_filters=128,
filter_size=(3, 3, 3),
axis=(2, 4, 5),
channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l8 = lasagne.layers.DropoutLayer(l7, p=0.5)
l_d3a = lasagne.layers.DenseLayer(l8, num_units=128)
l_d3b = lasagne.layers.DropoutLayer(l_d3a)
l_systole = lasagne.layers.DenseLayer(l_d3b, num_units=600, nonlinearity=lasagne.nonlinearities.softmax)
l_d3c = lasagne.layers.DenseLayer(l8, num_units=128)
l_d3d = lasagne.layers.DropoutLayer(l_d3c)
l_diastole = lasagne.layers.DenseLayer(l_d3d, num_units=600, nonlinearity=lasagne.nonlinearities.softmax)
return {
"inputs": {"sliced:data:ax": l0},
"outputs": {"systole:onehot": l_systole, "diastole:onehot": l_diastole},
"regularizable": {l_d3a: 0.25, l_d3c: 0.25, l_systole: 0.25, l_diastole: 0.25},
}
def _get_l_out(self, input_vars, multi_utt='ignored'):
check_options(self.options)
id_tag = (self.id + '/') if self.id else ''
input_var = input_vars[0]
context_vars = input_vars[1:]
l_in = InputLayer(shape=(None, self.seq_vec.max_len), input_var=input_var,
name=id_tag + 'desc_input')
l_in_embed = EmbeddingLayer(l_in, input_size=len(self.seq_vec.tokens),
output_size=self.options.listener_cell_size,
name=id_tag + 'desc_embed')
# Context repr has shape (batch_size, seq_len, context_len * repr_size)
l_context_repr, context_inputs = self.color_vec.get_input_layer(
context_vars,
recurrent_length=self.seq_vec.max_len,
cell_size=self.options.listener_cell_size,
context_len=self.context_len,
id=self.id
)
l_context_repr = reshape(l_context_repr, ([0], [1], self.context_len,
self.color_vec.output_size))
l_hidden_context = dimshuffle(l_context_repr, (0, 3, 1, 2), name=id_tag + 'shuffle_in')
for i in range(1, self.options.listener_hidden_color_layers + 1):
l_hidden_context = NINLayer(
l_hidden_context, num_units=self.options.listener_cell_size,
nonlinearity=NONLINEARITIES[self.options.listener_nonlinearity],
b=Constant(0.1),
name=id_tag + 'hidden_context%d' % i)
l_pool = FeaturePoolLayer(l_hidden_context, pool_size=self.context_len, axis=3,
pool_function=T.mean, name=id_tag + 'pool')
l_pool_squeezed = reshape(l_pool, ([0], [1], [2]), name=id_tag + 'pool_squeezed')
l_pool_shuffle = dimshuffle(l_pool_squeezed, (0, 2, 1), name=id_tag + 'shuffle_out')
l_concat = ConcatLayer([l_pool_shuffle, l_in_embed], axis=2,
name=id_tag + 'concat_inp_context')
cell = CELLS[self.options.listener_cell]
cell_kwargs = {
'grad_clipping': self.options.listener_grad_clipping,
'num_units': self.options.listener_cell_size,
}
if self.options.listener_cell == 'LSTM':
cell_kwargs['forgetgate'] = Gate(b=Constant(self.options.listener_forget_bias))
if self.options.listener_cell != 'GRU':
cell_kwargs['nonlinearity'] = NONLINEARITIES[self.options.listener_nonlinearity]
# l_rec1_drop = l_concat
l_rec1 = cell(l_concat, name=id_tag + 'rec1', **cell_kwargs)
if self.options.listener_dropout > 0.0:
l_rec1_drop = DropoutLayer(l_rec1, p=self.options.listener_dropout,
name=id_tag + 'rec1_drop')
else:
l_rec1_drop = l_rec1
l_rec2 = cell(l_rec1_drop, name=id_tag + 'rec2', only_return_final=True, **cell_kwargs)
if self.options.listener_dropout > 0.0:
l_rec2_drop = DropoutLayer(l_rec2, p=self.options.listener_dropout,
name=id_tag + 'rec2_drop')
else:
l_rec2_drop = l_rec2
l_rec2_drop = NINLayer(l_rec2_drop, num_units=self.options.listener_cell_size,
nonlinearity=None, name=id_tag + 'rec2_dense')
# Context is fed into the RNN as one copy for each time step; just use
# the first time step for output.
# Input shape: (batch_size, repr_size, seq_len, context_len)
# Output shape: (batch_size, repr_size, context_len)
l_context_nonrec = SliceLayer(l_hidden_context, indices=0, axis=2,
name=id_tag + 'context_nonrec')
l_pool_nonrec = SliceLayer(l_pool_squeezed, indices=0, axis=2,
name=id_tag + 'pool_nonrec')
# Output shape: (batch_size, repr_size, context_len)
l_sub = broadcast_sub_layer(l_pool_nonrec, l_context_nonrec,
feature_dim=self.options.listener_cell_size,
id_tag=id_tag)
# Output shape: (batch_size, repr_size * 2, context_len)
l_concat_sub = ConcatLayer([l_context_nonrec, l_sub], axis=1,
name=id_tag + 'concat_inp_context')
# Output shape: (batch_size, cell_size, context_len)
l_hidden = NINLayer(l_concat_sub, num_units=self.options.listener_cell_size,
nonlinearity=None, name=id_tag + 'hidden')
if self.options.listener_dropout > 0.0:
l_hidden_drop = DropoutLayer(l_hidden, p=self.options.listener_dropout,
name=id_tag + 'hidden_drop')
else:
l_hidden_drop = l_hidden
l_dot = broadcast_dot_layer(l_rec2_drop, l_hidden_drop,
feature_dim=self.options.listener_cell_size,
id_tag=id_tag)
l_dot_bias = l_dot # BiasLayer(l_dot, name=id_tag + 'dot_bias')
l_dot_clipped = NonlinearityLayer(
l_dot_bias,
nonlinearity=NONLINEARITIES[self.options.listener_nonlinearity],
name=id_tag + 'dot_clipped')
l_scores = NonlinearityLayer(l_dot_clipped, nonlinearity=softmax, name=id_tag + 'scores')
return l_scores, [l_in] + context_inputs
def multihead_attention(input_sequence, query,
key_sequence=None, mask_input=None,
num_heads=1,key_size=None,value_size=None,
attn_class=DotAttentionLayer, name='multihead_attn',
**kwargs):
"""
A convenience function that computes K attention "heads" in parallel and concatenates them.
Each "head" is based on num_heads linear transformations of input sequence, query, and keys
:param attn_class: what kind of attention layer to apply in multi-headed mode (Attention or DotAttention)
:param num heads: the amount of parallel "heads"
:param key_size: num units in attention query and key, defaults to key_sequence.shape[-1]
:param value_size: num units in attention values, defaults to input_sequence.shape[-1]
:param input_sequence: sequence of inputs to be processed with attention
:type input_sequence: lasagne.layers.Layer with shape [batch,seq_length,units]
:param query: single time-step state of decoder that is used as query (usually custom layer or lstm/gru/rnn hid)
If it matches input_sequence one-step size, query is used as is.
Otherwise, DotAttention is performed from DenseLayer(query,input_units,nonlinearity=None).
:type query: lasagne.layers.Layer with shape [batch,units]
:param key_sequence: a sequence of keys to compute dot_product with. By default, uses input_sequence instead.
:type key_sequence: lasagne.layers.Layer with shape [batch,seq_length,units] or None
:param mask_input: mask for input_sequence (like other lasagne masks). Default is no mask
:type mask_input: lasagne.layers.Layer with shape [batch,seq_length]
Heavily inspired by https://arxiv.org/abs/1706.03762 and http://bit.ly/2vsYX0R
"""
assert len(input_sequence.output_shape) == 3, "input_sequence must be a 3-dimensional (batch,time,units)"
assert len(query.output_shape) == 2, "query must be a 2-dimensional for single tick (batch,units)"
assert mask_input is None or len(
mask_input.output_shape) == 2, "mask_input must be 2-dimensional (batch,time) or None"
assert key_sequence is None or len(key_sequence.output_shape) == 3, "key_sequence must be 3-dimensional " \
"of shape (batch,time,units) or None"
key_sequence = key_sequence or input_sequence
key_size = key_size or key_sequence.output_shape[-1]
value_size = value_size or input_sequence.output_shape[-1]
def make_broadcasted_heads(incoming,head_size,name=None):
ndim = len(incoming.output_shape)
assert ndim in (2,3), "incoming must be 2-dimensional (query) or 3-dimensional (key or value)"
heads = DenseLayer(incoming,head_size*num_heads,nonlinearity=None,
num_leading_axes=ndim-1,name=name) #[batch,time,head_size*num_heads]
if ndim == 3:
heads = reshape(heads,([0],[1],head_size,num_heads), name=name) #[batch,time,head_size,num_heads]
broadcasted_heads = BroadcastLayer(heads, (0, 3), name=name) #[batch*heads,time,head_size]
else: #ndim == 2
heads = reshape(heads, ([0], head_size, num_heads), name=name) # [batch,head_size,num_heads]
broadcasted_heads = BroadcastLayer(heads, (0, 2), name=name) # [batch*heads, head_size]
return broadcasted_heads
query_heads = make_broadcasted_heads(query, key_size,name=name + "_query_heads")
value_heads = make_broadcasted_heads(input_sequence, value_size, name=name + "_value_heads")
if key_sequence is not None:
key_heads = make_broadcasted_heads(key_sequence, key_size, name=name + "_key_heads")
else:
key_heads = None
if mask_input is not None:
mask_heads = UpcastLayer(mask_input,broadcast_layer=query_heads)
else:
mask_heads = None
attn_heads = attn_class(value_heads,query_heads,key_sequence=key_heads,
mask_input=mask_heads,name=name,**kwargs) #[batch*heads,value_size]
attn_vectors = UnbroadcastLayer(attn_heads['attn'],broadcast_layer=query_heads) #[batch,value,heads]
attn_vectors = flatten(attn_vectors,outdim=2)
attn_probs = reshape(attn_heads['probs'],(-1,num_heads,[1])) #[batch,head,probs]
return {'attn': attn_vectors, #[batch, value*heads]
'probs': attn_probs}
def build_model():
#################
# Regular model #
#################
input_key = "sliced:data:ax:noswitch"
data_size = data_sizes[input_key]
l0 = InputLayer(data_size)
l0r = batch_norm(reshape(l0, (-1, 1, ) + data_size[1:]))
# (batch, channel, axis, time, x, y)
# convolve over time
l1 = batch_norm(ConvolutionOverAxisLayer(l0r, num_filters=4, filter_size=(3,), axis=(3,), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.0),
))
l1m = batch_norm(MaxPoolOverAxisLayer(l1, pool_size=(4,), axis=(3,)))
# convolve over x and y
l2a = batch_norm(ConvolutionOver2DAxisLayer(l1m, num_filters=8, filter_size=(3, 3),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.0),
))
l2b = batch_norm(ConvolutionOver2DAxisLayer(l2a, num_filters=8, filter_size=(3, 3),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.0),
))
l2m = batch_norm(MaxPoolOver2DAxisLayer(l2b, pool_size=(2, 2), axis=(4,5)))
# convolve over x, y, time
l3a = batch_norm(ConvolutionOver3DAxisLayer(l2m, num_filters=32, filter_size=(3, 3, 3),
axis=(3,4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
))
l3b = batch_norm(ConvolutionOver2DAxisLayer(l3a, num_filters=32, filter_size=(3, 3),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
))
l3m = batch_norm(MaxPoolOver2DAxisLayer(l3b, pool_size=(2, 2), axis=(4,5)))
# convolve over time
l4 = batch_norm(ConvolutionOverAxisLayer(l3m, num_filters=32, filter_size=(3,), axis=(3,), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
))
l4m = batch_norm(MaxPoolOverAxisLayer(l4, pool_size=(2,), axis=(2,)))
# maxpool over axis
l5 = batch_norm(MaxPoolOverAxisLayer(l3m, pool_size=(4,), axis=(2,)))
# convolve over x and y
l6a = batch_norm(ConvolutionOver2DAxisLayer(l5, num_filters=128, filter_size=(3, 3),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
))
l6b = batch_norm(ConvolutionOver2DAxisLayer(l6a, num_filters=128, filter_size=(3, 3),
axis=(4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
))
l6m = batch_norm(MaxPoolOver2DAxisLayer(l6b, pool_size=(2, 2), axis=(4,5)))
# convolve over time and x,y
l7 = ConvolutionOver3DAxisLayer(l6m, num_filters=128, filter_size=(3,3,3), axis=(3,4,5), channel=1,
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l8 = lasagne.layers.DropoutLayer(l7, p=0.5)
l_systole = CumSumLayer(lasagne.layers.DenseLayer(l8,
num_units=600,
nonlinearity=lasagne.nonlinearities.softmax))
l_diastole = CumSumLayer(lasagne.layers.DenseLayer(l8,
num_units=600,
nonlinearity=lasagne.nonlinearities.softmax))
return {
"inputs":{
input_key: l0
},
"outputs": {
"systole": l_systole,
"diastole": l_diastole,
},
"regularizable": {
}
}
def build_model():
###############
# Sunny model #
###############
l0_sunny = InputLayer(data_sizes["sunny"])
l1a_sunny = ConvLayer(
l0_sunny,
num_filters=32,
filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1b_sunny = ConvLayer(
l1a_sunny,
num_filters=32,
filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1c_sunny = ConvLayer(
l1b_sunny,
num_filters=32,
filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1f_sunny = ConvLayer(l1c_sunny,
num_filters=1,
filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
nonlinearity=lasagne.nonlinearities.sigmoid)
#l_sunny_segmentation = lasagne.layers.reshape(l1d_sunny, data_sizes["sunny"][:1] + l1d_sunny.output_shape[-2:])
l_sunny_segmentation = lasagne.layers.SliceLayer(l1f_sunny,
indices=0,
axis=1)
#################
# Regular model #
#################
l0 = InputLayer(data_sizes["sliced:data:ax"])
l0r = reshape(l0, (
-1,
1,
) + data_sizes["sliced:data:ax"][-2:])
# first do the segmentation steps
l1a = ConvLayer(l0r,
num_filters=32,
filter_size=(3, 3),
pad='same',
W=l1a_sunny.W,
b=l1a_sunny.b)
l1b = ConvLayer(l1a,
num_filters=32,
filter_size=(3, 3),
pad='same',
W=l1b_sunny.W,
b=l1b_sunny.b)
l1c = ConvLayer(l1b,
num_filters=64,
filter_size=(3, 3),
pad='same',
W=l1c_sunny.W,
b=l1c_sunny.b)
l1f = ConvLayer(l1c,
num_filters=1,
filter_size=(3, 3),
pad='same',
W=l1f_sunny.W,
b=l1f_sunny.b,
nonlinearity=lasagne.nonlinearities.sigmoid)
l_1r = reshape(l1f, data_sizes["sliced:data:ax"])
l_d3 = lasagne.layers.DenseLayer(l_1r, num_units=2)
l_systole = MuLogSigmaErfLayer(l_d3)
l_d3b = lasagne.layers.DenseLayer(l_1r, num_units=2)
l_diastole = MuLogSigmaErfLayer(l_d3b)
return {
"inputs": {
"sliced:data:ax": l0,
"sunny": l0_sunny,
},
"outputs": {
"systole": l_systole,
"diastole": l_diastole,
"segmentation": l_sunny_segmentation,
}
}
def build_model():
###############
# Sunny model #
###############
l0_sunny = InputLayer(data_sizes["sunny"])
l1a_sunny = ConvLayer(
l0_sunny,
num_filters=32,
filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1b_sunny = WideConv2DDNNLayer(
l1a_sunny,
num_filters=32,
filter_size=(3, 3),
skip=2,
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1c_sunny = WideConv2DDNNLayer(
l1b_sunny,
num_filters=32,
filter_size=(3, 3),
skip=4,
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1d_sunny = WideConv2DDNNLayer(
l1c_sunny,
num_filters=32,
filter_size=(3, 3),
skip=8,
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1e_sunny = WideConv2DDNNLayer(
l1d_sunny,
num_filters=32,
filter_size=(3, 3),
skip=16,
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1f_sunny = WideConv2DDNNLayer(l1e_sunny,
num_filters=1,
filter_size=(3, 3),
skip=32,
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
nonlinearity=lasagne.nonlinearities.sigmoid)
#l_sunny_segmentation = lasagne.layers.reshape(l1d_sunny, data_sizes["sunny"][:1] + l1d_sunny.output_shape[-2:])
l_sunny_segmentation = lasagne.layers.SliceLayer(l1f_sunny,
indices=0,
axis=1)
#################
# Regular model #
#################
l0 = InputLayer(data_sizes["sliced:data:ax"])
l0r = reshape(l0, (
-1,
1,
) + data_sizes["sliced:data:ax"][-2:])
# first do the segmentation steps
l1a = ConvLayer(l0r,
num_filters=32,
filter_size=(3, 3),
pad='same',
W=l1a_sunny.W,
b=l1a_sunny.b)
l1b = WideConv2DDNNLayer(l1a,
num_filters=32,
filter_size=(3, 3),
skip=2,
pad='same',
W=l1b_sunny.W,
b=l1b_sunny.b)
l1c = WideConv2DDNNLayer(l1b,
num_filters=64,
filter_size=(3, 3),
skip=4,
pad='same',
W=l1c_sunny.W,
b=l1c_sunny.b)
l1d = WideConv2DDNNLayer(l1c,
num_filters=64,
filter_size=(3, 3),
skip=8,
pad='same',
W=l1d_sunny.W,
b=l1d_sunny.b)
l1e = WideConv2DDNNLayer(l1d,
num_filters=32,
filter_size=(3, 3),
skip=16,
pad='same',
W=l1e_sunny.W,
b=l1e_sunny.b)
l1f = WideConv2DDNNLayer(l1e,
num_filters=1,
filter_size=(3, 3),
skip=32,
pad='same',
W=l1f_sunny.W,
b=l1f_sunny.b,
nonlinearity=lasagne.nonlinearities.sigmoid)
l_1r = reshape(l1f, data_sizes["sliced:data:ax"])
# returns (batch, time, 600) of probabilities
# TODO: it should also take into account resolution, etc.
volume_layer = GaussianApproximationVolumeLayer(l_1r)
# then use max and min over time for systole and diastole
l_systole = lasagne.layers.FlattenLayer(lasagne.layers.FeaturePoolLayer(
volume_layer,
pool_size=volume_layer.output_shape[1],
axis=1,
pool_function=T.min),
outdim=2)
l_diastole = lasagne.layers.FlattenLayer(lasagne.layers.FeaturePoolLayer(
volume_layer,
pool_size=volume_layer.output_shape[1],
axis=1,
pool_function=T.max),
outdim=2)
return {
"inputs": {
"sliced:data:ax": l0,
"sunny": l0_sunny,
},
"outputs": {
"systole": l_systole,
"diastole": l_diastole,
"segmentation": l_sunny_segmentation,
}
}
def build_sequence_dense_layer(input_var, input_layer, output_dim):
n_batch, n_time_steps, _ = input_var.shape
dense_layer = DenseLayer(reshape(input_layer, (-1, [2])),
num_units=output_dim,
nonlinearity=nonlinearities.softmax)
return reshape(dense_layer, (n_batch, n_time_steps, output_dim))
def build_model():
###############
# Sunny model #
###############
l0_sunny = InputLayer(data_sizes["sunny"])
l1a_sunny = ConvLayer(l0_sunny, num_filters=32, filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1b_sunny = ConvLayer(l1a_sunny, num_filters=32, filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1c_sunny = ConvLayer(l1b_sunny, num_filters=32, filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
)
l1f_sunny = ConvLayer(l1c_sunny, num_filters=1, filter_size=(3, 3),
pad='same',
W=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.1),
nonlinearity=lasagne.nonlinearities.sigmoid)
#l_sunny_segmentation = lasagne.layers.reshape(l1d_sunny, data_sizes["sunny"][:1] + l1d_sunny.output_shape[-2:])
l_sunny_segmentation = lasagne.layers.SliceLayer(l1f_sunny, indices=0, axis=1)
#################
# Regular model #
#################
l0 = InputLayer(data_sizes["sliced:data:ax"])
l0r = reshape(l0, (-1, 1, ) + data_sizes["sliced:data:ax"][-2:])
# first do the segmentation steps
l1a = ConvLayer(l0r, num_filters=32, filter_size=(3, 3),
pad='same',
W=l1a_sunny.W,
b=l1a_sunny.b)
l1b = ConvLayer(l1a, num_filters=32, filter_size=(3, 3),
pad='same',
W=l1b_sunny.W,
b=l1b_sunny.b)
l1c = ConvLayer(l1b, num_filters=64, filter_size=(3, 3),
pad='same',
W=l1c_sunny.W,
b=l1c_sunny.b)
l1f = ConvLayer(l1c, num_filters=1, filter_size=(3, 3),
pad='same',
W=l1f_sunny.W,
b=l1f_sunny.b,
nonlinearity=lasagne.nonlinearities.sigmoid)
l_1r = reshape(l1f, data_sizes["sliced:data:ax"])
l_d3 = lasagne.layers.DenseLayer(l_1r,
num_units=2)
l_systole = MuLogSigmaErfLayer(l_d3)
l_d3b = lasagne.layers.DenseLayer(l_1r,
num_units=2)
l_diastole = MuLogSigmaErfLayer(l_d3b)
return {
"inputs":{
"sliced:data:ax": l0,
"sunny": l0_sunny,
},
"outputs":{
"systole": l_systole,
"diastole": l_diastole,
"segmentation": l_sunny_segmentation,
}
}
