def _build_clockwork_block(self, units, period, input_shape):
output_shape = input_shape[:-1] + (units,)
pool = MaxPooling1D(1, period)
rnn = self.rnn_dtype(units=units, **self.rnn_kwargs)
unpool = UpSampling1D(period)
crop = Lambda(lambda x: self._crop(x[0], x[1]),
output_shape=output_shape[1:])
delay = Lambda(lambda x: self._delay(x),
output_shape=output_shape[1:])
concat = Concatenate()
block = (pool, rnn, unpool, crop, delay, concat)
pool.build(input_shape)
pool_output_shape = pool.compute_output_shape(input_shape)
rnn.build(pool_output_shape)
self._trainable_weights.extend(rnn.trainable_weights)
rnn_output_shape = rnn.compute_output_shape(pool_output_shape)
unpool.build(rnn_output_shape)
crop.build([unpool.compute_output_shape(rnn_output_shape), ()])
delay.build(output_shape)
concat.build([input_shape, output_shape])
return block, output_shape, \
concat.compute_output_shape([input_shape, output_shape])
alpha = K.softmax(e) # (batch_size, input_length)
# eqn 5
c = K.batch_dot(h, alpha, axes=1) # (batch_size, encoding_dim)
recurrence_result = K.expand_dims(
K.concatenate([c, y_i], axis=1),
dim=1) # (batch_size, 1, 2 * encoding_dim)
expanded_h = Input(shape=(1, 2 * encoding_dim),
name='expanded_h')
gru = Sequential([
GRU(output_dim,
return_sequences=False,
input_shape=(1, 2 * encoding_dim))
])
model = Model(input=[expanded_h],
output=[gru(expanded_h)]) # (batch_size, 1, output_dim)
return model(recurrence_result)
output, _ = theano.scan(recurrence,
sequences=K.permute_dimensions(y, [1, 0, 2]),
non_sequences=h)
layer = Lambda(lambda encoded_state: output,
output_shape=(batch_size, output_dim))
layer.build((input_length, encoding_dim))
print(K.eval(layer(h)))
# eqn 6
alpha = K.softmax(e) # (batch_size, input_length)
# eqn 5
c = K.batch_dot(h, alpha, axes=1) # (batch_size, encoding_dim)
recurrence_result = K.expand_dims(K.concatenate(
[c, y_i], axis=1), dim=1) # (batch_size, 1, 2 * encoding_dim)
expanded_h = Input(shape=(1, 2 * encoding_dim), name='expanded_h')
gru = Sequential([
GRU(output_dim,
return_sequences=False,
input_shape=(1, 2 * encoding_dim))
])
model = Model(input=[expanded_h],
output=[gru(expanded_h)]) # (batch_size, 1, output_dim)
return model(recurrence_result)
output, _ = theano.scan(recurrence,
sequences=K.permute_dimensions(y, [1, 0, 2]),
non_sequences=h)
layer = Lambda(lambda encoded_state: output,
output_shape=(batch_size, output_dim))
layer.build((input_length, encoding_dim))
print(K.eval(layer(h)))
def window(w, n):
w = w + list(reversed(w))
w = numpy.asarray(w)
layer = Lambda(lambda x: x * w)
layer.build((n, ))
return layer
class ResidualBlock(Layer):
def __init__(self,
dilation_rate,
nb_filters,
kernel_size,
padding,
activation='relu',
dropout_rate=0,
kernel_initializer='he_normal',
use_batch_norm=False,
last_block=True,
**kwargs):
# type: (int, int, int, str, str, float, str, bool, dict) -> None
"""Defines the residual block for the WaveNet TCN
Args:
x: The previous layer in the model
training: boolean indicating whether the layer should behave in training mode or in inference mode
dilation_rate: The dilation power of 2 we are using for this residual block
nb_filters: The number of convolutional filters to use in this block
kernel_size: The size of the convolutional kernel
padding: The padding used in the convolutional layers, 'same' or 'causal'.
activation: The final activation used in o = Activation(x + F(x))
dropout_rate: Float between 0 and 1. Fraction of the input units to drop.
kernel_initializer: Initializer for the kernel weights matrix (Conv1D).
use_batch_norm: Whether to use batch normalization in the residual layers or not.
kwargs: Any initializers for Layer class.
"""
self.dilation_rate = dilation_rate
self.nb_filters = nb_filters
self.kernel_size = kernel_size
self.padding = padding
self.activation = activation
self.dropout_rate = dropout_rate
self.use_batch_norm = use_batch_norm
self.kernel_initializer = kernel_initializer
self.last_block = last_block
super(ResidualBlock, self).__init__(**kwargs)
def _add_and_activate_layer(self, layer):
"""Helper function for building layer
Args:
layer: Appends layer to internal layer list and builds it based on the current output
shape of ResidualBlock. Updates current output shape.
"""
self.residual_layers.append(layer)
self.residual_layers[-1].build(self.res_output_shape)
self.res_output_shape = self.residual_layers[-1].compute_output_shape(
self.res_output_shape)
def build(self, input_shape):
with K.name_scope(
self.name
): # name scope used to make sure weights get unique names
self.residual_layers = list()
self.res_output_shape = input_shape
for k in range(2):
name = 'conv1D_{}'.format(k)
with K.name_scope(
name
): # name scope used to make sure weights get unique names
self._add_and_activate_layer(
Conv1D(filters=self.nb_filters,
kernel_size=self.kernel_size,
dilation_rate=self.dilation_rate,
padding=self.padding,
name=name,
kernel_initializer=self.kernel_initializer))
if self.use_batch_norm:
# TODO should be WeightNorm here, but using batchNorm instead
self._add_and_activate_layer(BatchNormalization())
self._add_and_activate_layer(Activation('relu'))
self._add_and_activate_layer(
SpatialDropout1D(rate=self.dropout_rate))
if not self.last_block:
# 1x1 conv to match the shapes (channel dimension).
name = 'conv1D_{}'.format(k + 1)
with K.name_scope(name):
# make and build this layer separately because it directly uses input_shape
self.shape_match_conv = Conv1D(
filters=self.nb_filters,
kernel_size=1,
padding='same',
name=name,
kernel_initializer=self.kernel_initializer)
else:
self.shape_match_conv = Lambda(lambda x: x, name='identity')
self.shape_match_conv.build(input_shape)
self.res_output_shape = self.shape_match_conv.compute_output_shape(
input_shape)
self.final_activation = Activation(self.activation)
self.final_activation.build(
self.res_output_shape) # probably isn't necessary
# this is done to force keras to add the layers in the list to self._layers
for layer in self.residual_layers:
self.__setattr__(layer.name, layer)
super(ResidualBlock, self).build(
input_shape) # done to make sure self.built is set True
def call(self, inputs, training=None):
"""
Returns: A tuple where the first element is the residual model tensor, and the second
is the skip connection tensor.
"""
x = inputs
for layer in self.residual_layers:
if isinstance(layer, SpatialDropout1D):
x = layer(x, training=training)
else:
x = layer(x)
x2 = self.shape_match_conv(inputs)
res_x = add([x2, x])
return [self.final_activation(res_x), x]
def compute_output_shape(self, input_shape):
return [self.res_output_shape, self.res_output_shape]
dim=1) # (batch_size, 1, 2 * encoding_dim)
return model(input_per_timesteps)
output, _ = theano.scan(recurrence,
sequences=K.permute_dimensions(y, [1, 0, 2]),
non_sequences=h)
return K.permute_dimensions(output, [1, 0, 2])
x = Input(shape=(input_length,), dtype='int32', name='x')
y = Input(shape=(output_length,), dtype='int32', name='y')
embed = Embedding(voc_size, hidden_dim, input_length=input_length)
decode = Lambda(decode_func,
output_shape=(batch_size, output_dim),
arguments={'y': embed(y)})
decode.build((input_length, hidden_dim))
gru = GRU(hidden_dim,
input_shape=(input_length, hidden_dim),
unroll=True,
return_sequences=True,
consume_less='gpu')
encoder = Sequential([embed])
for _ in range(depth):
encoder.add(gru)
hidden_state = encoder(x)
def model(train):
def decoder_unit():
return GRU(hidden_dim,
return_sequences=True,
def mse(y_true, y_pred):
cost = cost + K.mean(K.square(y_pred - y_true), axis=-1)
# Autoencoder (or) Denoising autoencoder with the comb denoising function
# Decoder begins here with the upsampling -- connecting to the last convolutional layer output of the noisy encoder
# NOTE: There is no scaling or shifting in the decoder implementation according to the paper. There is only batch normalization
# Getting the output from the final dense layer (dense_3) of the noisy encoder and passing it to the decoder module
lam = Lambda(lambda x: x,
output_shape=(ns_enc.get_layer('dense_3').output_shape[1], w, h,
1))
lam.build((ns_enc.get_layer('dense_3').output_shape[1], w, h, 1))
x = K.reshape(
ns_enc.get_layer('dense_3').output,
(ns_enc.get_layer('dense_3').output_shape[1], w, h, 1))
out = lam(x)
cde1 = Conv2DTranspose(32, 3, 3, name="conv_de_1")(out)
btde1 = BatchNormalization(center=False, scale=False, name="btde1")(cde1)
a1 = LeakyReLU(name="actde1")(btde1)
#denoising function comb call if denoising autoencoder
cde2 = Conv2DTranspose(64, 3, 3, name="conv_de_2")(a1)
btde2 = BatchNormalization(center=False, scale=False, name="btde2")(cde2)
a2 = LeakyReLU(name="actde2")(btde2)
#denoising function comb call if denoising autoencoder
cde3 = Conv2DTranspose(64, 3, 3, name="conv_de_3")(a2)
btde3 = BatchNormalization(center=False, scale=False, name="btde3")(cde3)
class ClockworkRNN(Layer):
"""Clockwork RNN ([Koutnik et al., 2014](https://arxiv.org/abs/1402.3511)).
Constructs a CW-RNN from RNNs of a given type.
# Arguments
periods: List of positive integers. The periods of each internal RNN.
units_per_period: Positive integer or list of positive integers.
Number of units for each internal RNN. If list, it must have the
same length as `periods`.
input_shape: Shape of the input data.
output_units: Positive integer. Dimensionality of the output space.
output_activation: String or callable. Activation function to use. If
you don't specify anything, no activation is applied (i.e.,
"linear" activation: `a(x) = x`).
return_sequences: Boolean (default False). Whether to return the last
output in the output sequence, or the full sequence.
sort_ascending: Boolean (default False). Whether to sort the periods
in ascending or descending order (default, as in the original
paper).
include_top: Whether to include the fully-connected layer at the top
of the network.
dense_kwargs: Dictionary. Optional arguments for the trailing Dense
unit (`activation` and `units` keys will be ignored).
rnn_dtype: The type of RNN to use as clockwork layer. Can be a string
("SimpleRNN", "GRU", "LSTM", "CuDNNGRU", "CuDNNLSTM") or any RNN
subclass.
rnn_kwargs: Dictionary. Optional arguments for the internal RNNs
(`return_sequences` and `return_state` will be ignored).
"""
def __init__(self, periods,
units_per_period,
output_units,
output_activtion='linear',
return_sequences=False,
sort_ascending=False,
include_top=True,
dense_kwargs=None,
rnn_dtype="SimpleRNN",
rnn_kwargs=None,
**kwargs):
if type(rnn_dtype) is str:
self.rnn_dtype = getattr(layers, rnn_dtype)
else:
self.rnn_dtype = rnn_dtype
ClockworkRNN.__name__ = "Clockwork" + self.rnn_dtype.__name__
super(ClockworkRNN, self).__init__(**kwargs)
if type(units_per_period) is list:
self.units_per_period = units_per_period
else:
self.units_per_period = [units_per_period] * len(periods)
self.periods = periods
self.rnn_kwargs = rnn_kwargs or {}
self.rnn_kwargs['return_sequences'] = True
self.rnn_kwargs['return_state'] = False
self.rnn_kwargs.pop("units", True)
self.dense_kwargs = dense_kwargs or {}
self.dense_kwargs['activation'] = output_activtion
self.dense_kwargs['units'] = output_units
self.include_top = include_top
self.return_sequences = return_sequences
self.sort_ascending = sort_ascending
self.blocks = []
def build(self, input_shape):
last_shape = input_shape
output_shapes = []
for period, units in sorted(zip(self.periods, self.units_per_period),
reverse=not self.sort_ascending):
block, output_shape, last_shape = self._build_clockwork_block(
units, period, last_shape)
output_shapes.append(output_shape)
self.blocks.append(block)
self.concat_all = Concatenate()
self.concat_all.build(output_shapes)
last_shape = self.concat_all.compute_output_shape(output_shapes)
if not self.return_sequences:
self.lambda_last = Lambda(lambda x: x[:, -1])
self.lambda_last.build(last_shape)
last_shape = self.lambda_last.compute_output_shape(last_shape)
if self.include_top:
if self.return_sequences:
self.dense = TimeDistributed(Dense(**self.dense_kwargs))
else:
self.dense = Dense(**self.dense_kwargs)
self.dense.build(last_shape)
self._trainable_weights.extend(self.dense.trainable_weights)
last_shape = self.dense.compute_output_shape(last_shape)
super(ClockworkRNN, self).build(input_shape)
def call(self, x):
rnns = []
to_next_block = x
for block in self.blocks:
to_dense, to_next_block = self._call_clockwork_block(
to_next_block, *block)
rnns.append(to_dense)
out = self.concat_all(rnns)
if not self.return_sequences:
out = self.lambda_last(out)
if self.include_top:
out = self.dense(out)
return out
def compute_output_shape(self, input_shape):
if self.include_top:
out_dim = self.dense_kwargs['units']
else:
out_dim = np.sum(self.units_per_period)
if self.return_sequences:
return input_shape[:-1] + (out_dim,)
else:
return input_shape[:-2] + (out_dim,)
def _delay(self, x):
return K.temporal_padding(x, (1, 0))[:, :-1]
def _crop(self, x, timesteps):
return x[:, :K.cast(timesteps, "int32")]
def _build_clockwork_block(self, units, period, input_shape):
output_shape = input_shape[:-1] + (units,)
pool = MaxPooling1D(1, period)
rnn = self.rnn_dtype(units=units, **self.rnn_kwargs)
unpool = UpSampling1D(period)
crop = Lambda(lambda x: self._crop(x[0], x[1]),
output_shape=output_shape[1:])
delay = Lambda(lambda x: self._delay(x),
output_shape=output_shape[1:])
concat = Concatenate()
block = (pool, rnn, unpool, crop, delay, concat)
pool.build(input_shape)
pool_output_shape = pool.compute_output_shape(input_shape)
rnn.build(pool_output_shape)
self._trainable_weights.extend(rnn.trainable_weights)
rnn_output_shape = rnn.compute_output_shape(pool_output_shape)
unpool.build(rnn_output_shape)
crop.build([unpool.compute_output_shape(rnn_output_shape), ()])
delay.build(output_shape)
concat.build([input_shape, output_shape])
return block, output_shape, \
concat.compute_output_shape([input_shape, output_shape])
def _call_clockwork_block(self, x, pool, rnn, unpool, crop, delay, concat):
pooled = pool(x)
rnn_out = rnn(pooled)
unpooled = unpool(rnn_out)
to_dense = crop([unpooled, K.shape(x)[1]])
delayed = delay(to_dense)
to_next_block = concat([x, delayed])
return to_dense, to_next_block
def get_config(self):
config = super(ClockworkRNN, self).get_config()
config['units_per_period'] = self.units_per_period
config['periods'] = self.periods
config['rnn_dtype'] = self.rnn_dtype.__name__
config['rnn_kwargs'] = self.rnn_kwargs
config['dense_kwargs'] = self.dense_kwargs
config['include_top'] = self.include_top
config['return_sequences'] = self.return_sequences
config['sort_ascending'] = self.sort_ascending
return config
def get_vnet_gen_model():
inputs = Input((1, num_slices, img_width, img_height))
# output 128 x 128 x 16 x 16
conv1 = Conv3D(16, 5, 5, 5, activation='relu', border_mode='same')(inputs)
# output 128 x 128 x 16 x 16
m0 = merge([inputs, inputs], mode='concat', concat_axis=1)
for i in range(14):
m0 = merge([inputs, m0], mode='concat', concat_axis=1)
m1 = merge([m0, conv1], output_shape=(None, 16, 16, 128, 128), mode='sum')
lam = Lambda(lambda x: x, output_shape=(16, 16, 128, 128))
lam.build((16, 16, 128, 128))
out = lam(m1)
# output 64 x 64 x 8 x 16
pool1 = MaxPooling3D(pool_size=(2, 2, 2))(out)
# output 64 x 64 x 8 x 32
conv2 = Conv3D(32, 5, 5, 5, activation='relu', border_mode='same')(pool1)
# output 64 x 64 x 8 x 32
conv2 = Conv3D(32, 5, 5, 5, activation='relu', border_mode='same')(conv2)
# output 64 x 64 x 8 x 32
m0 = merge([pool1, pool1], mode='concat', concat_axis=1)
#print m0.shape
#for i in range(14):
# m0 = merge([pool1, m0], mode='concat', concat_axis=1)
# print m0.shape
# output 64 x 64 x 8 x 32
m2 = merge([m0, conv2], mode='sum')
#lam = Lambda(lambda x: x, output_shape=(16, 16, 128, 128))
#lam.build((16, 16, 128, 128))
#out = lam(m2)
# output 64 x 64 x 8 x 64
#m2 = merge([pool1, conv2], mode='sum')
# output 32 x 32 x 4 x 32
pool2 = MaxPooling3D(pool_size=(2, 2, 2))(m2)
# output 32 x 32 x 4 x 64
conv3 = Conv3D(64, 5, 5, 5, activation='relu', border_mode='same')(pool2)
# output 32 x 32 x 4 x 64
conv3 = Conv3D(64, 5, 5, 5, activation='relu', border_mode='same')(conv3)
# output 32 x 32 x 4 x 64
conv3 = Conv3D(64, 5, 5, 5, activation='relu', border_mode='same')(conv3)
# output 32 x 32 x 4 x 64
m0 = merge([pool2, pool2], mode='concat', concat_axis=1)
# output 32 x 32 x 4 x 64
m3 = merge([m0, conv3], mode='sum')
# output 16 x 16 x 2 x 64
pool3 = MaxPooling3D(pool_size=(2, 2, 2))(m3)
conv4 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(pool3)
conv4 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(conv4)
# output 16 x 16 x 2 x 128
conv4 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(conv4)
# output 16 x 16 x 2 x 128
m0 = merge([pool3, pool3], mode='concat', concat_axis=1)
# output 16 x 16 x 2 x 128
m4 = merge([m0, conv4], mode='sum')
# output 8 x 8 x 1 x 128
pool4 = MaxPooling3D(pool_size=(2, 2, 2))(m4)
conv5 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(pool4)
conv5 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(conv5)
# output 8 x 8 x 1 x 256
conv5 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(conv5)
# output 8 x 8 x 1 x 256
m0 = merge([pool4, pool4], mode='concat', concat_axis=1)
# output 8 x 8 x 1 x 256
m5 = merge([m0, conv5], mode='sum')
# output 16 x 16 x 2 x 256
up5 = UpSampling3D(size=(2, 2, 2))(m5)
# output 16 x 16 x 2 x 384
m0 = merge([m4, up5], mode='concat', concat_axis=1)
conv6 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(m0)
conv6 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(conv6)
# output 16 x 16 x 2 x 256
conv6 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(conv6)
# output 16 x 16 x 2 x 256
m6 = merge([up5, conv6], mode='sum')
# output 32 x 32 x 4 x 256
up6 = UpSampling3D(size=(2, 2, 2))(m6)
conv7 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(up6)
# output 32 x 32 x 4 x 192
m0 = merge([m3, conv7], mode='concat', concat_axis=1)
conv7 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(m0)
# output 32 x 32 x 4 x 128
conv7 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(conv7)
# output 32 x 32 x 4 x 256
m0 = merge([conv7, conv7], mode='concat', concat_axis=1)
# output 32 x 32 x 4 x 256
m7 = merge([up6, m0], mode='sum')
# output 64 x 64 x 8 x 256
up7 = UpSampling3D(size=(2, 2, 2))(m7)
conv8 = Conv3D(64, 5, 5, 5, activation='relu', border_mode='same')(up7)
# output 64 x 64 x 8x 96
m0 = merge([m2, conv8], mode='concat', concat_axis=1)
# output 64 x 64 x 8 x 64
conv8 = Conv3D(64, 5, 5, 5, activation='relu', border_mode='same')(m0)
# output 64 x 64 x 8 x 128
m0 = merge([conv8, conv8], mode='concat', concat_axis=1)
# output 64 x 64 x 8 x 256
m0 = merge([m0, m0], mode='concat', concat_axis=1)
# output 64 x 64 x 8 x 256
m8 = merge([up7, m0], mode='sum')
# output 128 x 128 x 16 x 256
up8 = UpSampling3D(size=(2, 2, 2))(m8)
# output 128 x 128 x 16 x 32
conv9 = Conv3D(32, 5, 5, 5, activation='relu', border_mode='same')(up8)
# output 128 x 128 x 16 x 48
m0 = merge([m1, conv9], mode='concat', concat_axis=1)
# output 128 x 128 x 16 x 96
m0 = merge([m0, m0], mode='concat', concat_axis=1)
# output 128 x 128 x 16 x 192
m0 = merge([m0, m0], mode='concat', concat_axis=1)
# output 128 x 128 x 16 x 64
m1 = merge([conv9, conv9], mode='concat', concat_axis=1)
m2 = merge([m0, m1], mode='concat', concat_axis=1)
# output 128 x 128 x 16 x 256
m9 = merge([up8, m2], mode='sum')
conv10 = Conv3D(1, 1, 1, 1, activation='sigmoid', border_mode='same')(m9)
model = Model(input=inputs, output=conv10)
model.compile(optimizer=Adam(lr=1.0e-5),
loss=dice_coef_loss,
metrics=[dice_coef])
model.summary()
return model
