def separable_resnet_block(n_filters, input, dilation_rate=(1, 1)):
"""
Puts together a ResNet block with two _separable_ convolutions.
:param n_filters: Number of filters ("depth") of this convolution
:param input: Keras functional object representing input to this resnet block.
:param dilation_rate: Tuple of 2 integers, specifying the dilation rate for dilated convolution.
:return: Tensor, representing the output of this block.
"""
n_in = GroupNormalization(groups=n_filters // 4)(input)
a_in = Activation(tf.nn.leaky_relu)(n_in)
c1 = SeparableConv2D(filters=n_filters,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=dilation_rate,
padding='same',
use_bias=False)(a_in)
n1 = GroupNormalization(groups=n_filters // 4)(c1)
a1 = Activation(tf.nn.leaky_relu)(n1)
c2 = SeparableConv2D(filters=n_filters,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=dilation_rate,
padding='same',
use_bias=False)(a1)
s = Add()([input, c2])
return s
def resnet_block(n_filters, input, dilation_rate=(1, 1)):
"""
Pre-activation ResNet block with group normalization.
:param n_filters: Number of filters ("depth") of this convolution
:param input: Keras functional object representing input to this resnet block.
:return: Keras functional object representing output of this resnet block.
"""
n_in = GroupNormalization(groups=n_filters // 4)(input)
a_in = Activation(tf.nn.leaky_relu)(n_in)
c1 = Conv2D(filters=n_filters,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=dilation_rate,
padding='same',
use_bias=False)(a_in)
n1 = GroupNormalization(groups=n_filters // 4)(c1)
a1 = Activation(tf.nn.leaky_relu)(n1)
c2 = Conv2D(filters=n_filters,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=dilation_rate,
padding='same',
use_bias=False)(a1)
s = Add()([input, c2])
return s
def getModel(shape, loss='mse', optimizer='adam'):
# https://devblogs.nvidia.com/deep-learning-self-driving-cars/
NModel = Sequential()
# Normalization Layer
NModel.add(Lambda(lambda x: (x / 127.5) - 1., input_shape=shape))
# Convolutional Layer 1
NModel.add(Conv2D(filters=24, kernel_size=5, strides=(2, 2)))
# Group normalization module for keras.
# Downloaded from https://github.com/titu1994/Keras-Group-Normalization
NModel.add(GroupNormalization(groups=12, axis=-1))
NModel.add(Activation('relu'))
# Convolutional Layer 2
NModel.add(Conv2D(filters=36, kernel_size=5, strides=(2, 2)))
NModel.add(GroupNormalization(groups=18, axis=-1))
NModel.add(Activation('relu'))
# Convolutional Layer 3
NModel.add(Conv2D(filters=48, kernel_size=5, strides=(2, 2)))
NModel.add(GroupNormalization(groups=12, axis=-1))
NModel.add(Activation('relu'))
# Convolutional Layer 4
NModel.add(Conv2D(filters=64, kernel_size=3, strides=(1, 1)))
NModel.add(GroupNormalization(groups=32, axis=-1))
NModel.add(Activation('relu'))
# Convolutional Layer 5
NModel.add(Conv2D(filters=64, kernel_size=3, strides=(1, 1)))
NModel.add(GroupNormalization(groups=32, axis=-1))
NModel.add(Activation('relu'))
# Flatten Layers
NModel.add(Flatten())
# Fully Connected Layer 1
NModel.add(Dense(100))
NModel.add(Activation('relu'))
# Fully Connected Layer 2
NModel.add(Dense(50))
NModel.add(Activation('relu'))
# Fully Connected Layer 3
NModel.add(Dense(10))
NModel.add(Activation('relu'))
# Output Layer
NModel.add(Dense(1))
NModel.compile(loss=loss, optimizer=optimizer)
return NModel
def test_groupnorm_mode_twice():
# This is a regression test for issue #4881 with the old
# batch normalization functions in the Theano backend.
model = Sequential()
model.add(GroupNormalization(input_shape=(10, 5, 5), axis=1, groups=2))
model.add(GroupNormalization(input_shape=(10, 5, 5), axis=1, groups=2))
model.compile(loss='mse', optimizer='sgd')
x = np.random.normal(loc=5.0, scale=10.0, size=(20, 10, 5, 5))
model.fit(x, x, epochs=1, verbose=0)
model.predict(x)
def block_5(self):
"""
Fifth block of VGG-16.
input: feature map of dimension 16x16x512
output: feature map of dimension 8x8x512
"""
with tf.variable_scope("block_5", reuse=tf.AUTO_REUSE):
tf.set_random_seed(self.seed)
## first convolution:
conv_1 = self.conv2d(self.block_4, self.VGG16_weights[20],
self.VGG16_weights[21], "conv_1")
relu_1 = tf.nn.relu(conv_1)
gnorm_1 = GroupNormalization(groups=32, axis=-1,
epsilon=0.1)(relu_1)
drop_1 = tf.nn.dropout(gnorm_1, self.prob)
## second convolution:
conv_2 = self.conv2d(drop_1, self.VGG16_weights[22],
self.VGG16_weights[23], "conv_2")
relu_2 = tf.nn.relu(conv_2)
gnorm_2 = GroupNormalization(groups=32, axis=-1,
epsilon=0.1)(relu_2)
drop_2 = tf.nn.dropout(gnorm_2, self.prob)
## third convolution:
conv_3 = self.conv2d(drop_2, self.VGG16_weights[24],
self.VGG16_weights[25], "conv_3")
relu_3 = tf.nn.relu(conv_3)
gnorm_3 = GroupNormalization(groups=32, axis=-1,
epsilon=0.1)(relu_3)
drop_3 = tf.nn.dropout(gnorm_3, self.prob)
# max pooling:
pool_1 = tf.nn.max_pool(drop_3,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
return pool_1
def setup_disparity_refinement_net(self):
# Input image and high-res disparity map are the same size
img_in = Input(shape=(self.image_shape[0], self.image_shape[1], 32))
disp_in = Input(shape=(self.image_shape[0], self.image_shape[1], 1))
# Image branch
img_c1 = Conv2D(filters=16,
kernel_size=(3, 3),
strides=(1, 1),
padding='same')(img_in)
img_bn1 = GroupNormalization(groups=4)(img_c1)
img_a1 = Activation(tf.nn.leaky_relu)(img_bn1)
img_rn1 = resnet_block(n_filters=16, input=img_a1)
img_rn2 = resnet_block(n_filters=16,
dilation_rate=(2, 2),
input=img_rn1)
# Disparity branch
disp_c1 = Conv2D(filters=16,
kernel_size=(3, 3),
strides=(1, 1),
padding='same')(disp_in)
disp_bn1 = GroupNormalization(groups=4)(disp_c1)
disp_a1 = Activation(tf.nn.leaky_relu)(disp_bn1)
disp_rn1 = resnet_block(n_filters=16, input=disp_a1)
disp_rn2 = resnet_block(n_filters=16,
dilation_rate=(2, 2),
input=disp_rn1)
# Glue these together along feature axis
ct1 = Concatenate(axis=-1)([img_rn2, disp_rn2])
# Some more processing
ref_rn1 = resnet_block(n_filters=32, dilation_rate=(4, 4), input=ct1)
ref_rn2 = resnet_block(n_filters=32,
dilation_rate=(8, 8),
input=ref_rn1)
ref_rn3 = resnet_block(n_filters=32, input=ref_rn2)
ref_rn4 = resnet_block(n_filters=32, input=ref_rn3)
# Reduce to single channel output (disparity residual)
disp_resid_out = Conv2D(filters=1,
kernel_size=(3, 3),
strides=(1, 1),
padding='same')(ref_rn4)
self.disparity_refinement_model = Model([img_in, disp_in],
disp_resid_out)
def _resnet_bottleneck_block(input, filters, k=1, strides=(1, 1)):
''' Adds a pre-activation resnet block with bottleneck layers
Args:
input: input tensor
filters: number of output filters
k: width factor
strides: strides of the convolution layer
Returns: a keras tensor
'''
init = input
channel_axis = 1 if K.image_data_format() == "channels_first" else -1
bottleneck_expand = 4
x = GroupNormalization(axis=channel_axis)(input)
x = Activation('relu')(x)
if strides != (1, 1) or init._keras_shape[
channel_axis] != bottleneck_expand * filters * k:
init = Conv2D(bottleneck_expand * filters * k, (1, 1),
padding='same',
kernel_initializer='he_normal',
use_bias=False,
strides=strides)(x)
x = Conv2D(filters * k, (1, 1),
padding='same',
kernel_initializer='he_normal',
use_bias=False)(x)
x = GroupNormalization(axis=channel_axis)(x)
x = Activation('relu')(x)
x = Conv2D(filters * k, (3, 3),
padding='same',
kernel_initializer='he_normal',
use_bias=False,
strides=strides)(x)
x = GroupNormalization(axis=channel_axis, gamma_initializer='zeros')(x)
x = Activation('relu')(x)
x = Conv2D(bottleneck_expand * filters * k, (1, 1),
padding='same',
kernel_initializer='he_normal',
use_bias=False)(x)
m = add([x, init])
return m
def residual_block(input_tensor, num_of_filters, kernel_size, group_filters,
residual_name):
"""
This function is used to create a residual block as proposed in the paper 'Vehicle Instance Segmentation From Aerial Image
and Video Using a Multitask Learning Residual Fully Convolutional Network' by Mou L., Zhu X.X. (2018).
Inputs:
- input_tensor: Input to the residual block
- num_of_filters: Number of feature maps to be generated for the residual block
- kernel_size: Size of convolutional kernel to be used for the residual block
- group_filters: Number of groups to be used for group normalization
- residual_name: Prefix to be appended to the name of each convolutional layer in the residual block
Outputs:
- residual_output: Output of the residual block
"""
conv_1 = Conv2D(num_of_filters, (kernel_size, kernel_size),
padding='same',
name=residual_name + '_conv_1')(input_tensor)
conv_1_gn = GroupNormalization(groups=group_filters, axis=-1,
epsilon=0.1)(conv_1)
conv_1_act = Activation('relu')(conv_1_gn)
conv_2 = Conv2D(num_of_filters, (kernel_size, kernel_size),
padding='same',
name=residual_name + '_conv_2')(conv_1_act)
conv_2_gn = GroupNormalization(groups=group_filters, axis=-1,
epsilon=0.1)(conv_2)
conv_2_act = Activation('relu')(conv_2_gn)
conv_3 = Conv2D(num_of_filters, (kernel_size, kernel_size),
padding='same',
name=residual_name + '_conv_3')(conv_2_act)
conv_3_gn = GroupNormalization(groups=group_filters, axis=-1,
epsilon=0.1)(conv_3)
conv_res = Conv2D(num_of_filters, (kernel_size, kernel_size),
padding='same',
name=residual_name + '_conv_residual')(input_tensor)
conv_res_gn = GroupNormalization(groups=group_filters,
axis=-1,
epsilon=0.1)(conv_res)
residual_output = Add()([conv_3_gn, conv_res_gn])
return residual_output
def feature_extract(inputs):
x = tf.keras.layers.Conv2D(64,
kernel_size=3,
strides=1,
padding="same",
use_bias=False)(inputs)
x = GroupNormalization(32)(x)
x = tf.keras.layers.Activation(tf.keras.activations.relu)(x)
c1 = tf.keras.layers.MaxPooling2D(pool_size=3, strides=2,
padding="same")(x)
c2 = _bottleneck(c1, 64, strides=1)
c3 = _bottleneck(c2, 128, strides=2)
c4 = _bottleneck(c3, 256, strides=2)
c5 = _bottleneck(c4, 512, strides=2)
p5 = tf.keras.layers.Conv2D(256, kernel_size=1, strides=1)(c5)
x = tf.keras.layers.Conv2D(256, kernel_size=1, strides=1)(c4)
p4 = tf.keras.layers.Add()(
[tf.keras.layers.UpSampling2D(size=(2, 2))(p5), x])
p4 = tf.keras.layers.Conv2D(128, kernel_size=3, strides=1,
padding="same")(p4)
x = tf.keras.layers.Conv2D(128, kernel_size=1, strides=1)(c3)
p3 = tf.keras.layers.Add()(
[tf.keras.layers.UpSampling2D(size=(2, 2))(p4), x])
p3 = tf.keras.layers.Conv2D(128, kernel_size=3, strides=1,
padding="same")(p3)
return p3
def test_shared_groupnorm():
'''Test that a GN layer can be shared
across different data streams.
'''
# Test single layer reuse
bn = GroupNormalization(input_shape=(10, ), groups=2)
x1 = Input(shape=(10, ))
bn(x1)
x2 = Input(shape=(10, ))
y2 = bn(x2)
x = np.random.normal(loc=5.0, scale=10.0, size=(2, 10))
model = Model(x2, y2)
assert len(model.updates) == 0
model.compile('sgd', 'mse')
model.train_on_batch(x, x)
# Test model-level reuse
x3 = Input(shape=(10, ))
y3 = model(x3)
new_model = Model(x3, y3)
assert len(model.updates) == 0
new_model.compile('sgd', 'mse')
new_model.train_on_batch(x, x)
def getMode():
input_shape = (MAX_LEN, )
inputs = Input(shape=input_shape)
emb_inputs = Embedding(NB_WORDS,
EMBEDDING_DIM,
weights=[embedding_matrix],
trainable=True)(inputs)
convLayer = Conv1D(filters=16,
kernel_size=4,
activation='elu',
input_shape=input_shape,
kernel_regularizer=regularizers.l2(1e-4),
bias_regularizer=regularizers.l2(1e-4))(emb_inputs)
normalizationLayer = GroupNormalization(groups=4, axis=-1)(convLayer)
poolingLayer = MaxPooling1D(pool_size=4)(normalizationLayer)
flattenLayer = Flatten()(poolingLayer)
dropoutLayer = Dropout(0.25)(flattenLayer)
denseLayer = Dense(32,
activation='elu',
kernel_regularizer=regularizers.l2(1e-4),
bias_regularizer=regularizers.l2(1e-4))(dropoutLayer)
outLayer = Dense(1, activation='sigmoid')(denseLayer)
model = Model(inputs=inputs, outputs=outLayer)
model.compile(loss='binary_crossentropy',
optimizer=SGD(momentum=0.95, lr=0.005),
metrics=[binary_accuracy])
print(model.summary())
return model
def test_groupnorm_correctness_2d_different_groups():
norm1 = GroupNormalization(axis=1, input_shape=(10, 6), groups=2)
norm2 = GroupNormalization(axis=1, input_shape=(10, 6), groups=1)
norm3 = GroupNormalization(axis=1, input_shape=(10, 6), groups=10)
model = Sequential()
model.add(norm1)
model.compile(loss='mse', optimizer='rmsprop')
# centered on 5.0, variance 10.0
x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10, 6))
model.fit(x, x, epochs=5, verbose=0)
out = model.predict(x)
out -= np.reshape(K.eval(norm1.beta), (1, 10, 1))
out /= np.reshape(K.eval(norm1.gamma), (1, 10, 1))
assert_allclose(out.mean(axis=(0, 2)), 0.0, atol=1.1e-1)
assert_allclose(out.std(axis=(0, 2)), 1.0, atol=1.1e-1)
model = Sequential()
model.add(norm2)
model.compile(loss='mse', optimizer='rmsprop')
# centered on 5.0, variance 10.0
x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10, 6))
model.fit(x, x, epochs=5, verbose=0)
out = model.predict(x)
out -= np.reshape(K.eval(norm2.beta), (1, 10, 1))
out /= np.reshape(K.eval(norm2.gamma), (1, 10, 1))
assert_allclose(out.mean(axis=(0, 2)), 0.0, atol=1.1e-1)
assert_allclose(out.std(axis=(0, 2)), 1.0, atol=1.1e-1)
model = Sequential()
model.add(norm3)
model.compile(loss='mse', optimizer='rmsprop')
# centered on 5.0, variance 10.0
x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10, 6))
model.fit(x, x, epochs=5, verbose=0)
out = model.predict(x)
out -= np.reshape(K.eval(norm3.beta), (1, 10, 1))
out /= np.reshape(K.eval(norm3.gamma), (1, 10, 1))
assert_allclose(out.mean(axis=(0, 2)), 0.0, atol=1.1e-1)
assert_allclose(out.std(axis=(0, 2)), 1.0, atol=1.1e-1)
def block_4(self):
"""
Fourth block of VGG-16 with max-pooling layer removed.
input: feature map of dimension 28x28x256
output: feature map of dimension 28x28x512
"""
with tf.variable_scope("block_4", reuse=tf.AUTO_REUSE):
tf.set_random_seed(self.seed)
## first convolution:
conv_1 = self.conv2d(self.block_3, self.VGG16_weights[14],
self.VGG16_weights[15], "conv_1")
relu_1 = tf.nn.relu(conv_1)
gnorm_1 = GroupNormalization(groups=32, axis=-1,
epsilon=0.1)(relu_1)
drop_1 = tf.nn.dropout(gnorm_1, self.prob)
## second convolution:
conv_2 = self.conv2d(drop_1, self.VGG16_weights[16],
self.VGG16_weights[17], "conv_2")
relu_2 = tf.nn.relu(conv_2)
gnorm_2 = GroupNormalization(groups=32, axis=-1,
epsilon=0.1)(relu_2)
drop_2 = tf.nn.dropout(gnorm_2, self.prob)
## third convolution:
conv_3 = self.conv2d(drop_2, self.VGG16_weights[18],
self.VGG16_weights[19], "conv_3")
gnorm_3 = GroupNormalization(groups=32, axis=-1,
epsilon=0.1)(conv_3)
relu_3 = tf.nn.relu(gnorm_3)
return relu_3
def block(x, channels):
return tf.keras.Sequential([
tf.keras.layers.Conv2D(channels,
kernel_size=3,
strides=1,
padding="same"),
GroupNormalization(groups=channels // 4),
tf.keras.layers.Activation(tf.keras.activations.relu),
tf.keras.layers.MaxPooling2D((2, 1), strides=(2, 1)),
])(x)
def block_1(self):
"""
First block of VGG-16.
input: RGB image of dimension 224x224x3
output: feature map of dimension 112x112x128
"""
with tf.variable_scope("block_1", reuse=tf.AUTO_REUSE):
tf.set_random_seed(self.seed)
## first convolution:
conv_1 = self.conv2d(self.input_image, self.VGG16_weights[0],
self.VGG16_weights[1], "conv_1")
relu_1 = tf.nn.relu(conv_1)
gnorm_1 = GroupNormalization(groups=32, axis=-1,
epsilon=0.1)(relu_1)
drop_1 = tf.nn.dropout(gnorm_1, self.prob)
## second convolution:
conv_2 = self.conv2d(drop_1, self.VGG16_weights[2],
self.VGG16_weights[3], "conv_2")
relu_2 = tf.nn.relu(conv_2)
gnorm_2 = GroupNormalization(groups=32, axis=-1,
epsilon=0.1)(relu_2)
drop_2 = tf.nn.dropout(gnorm_2, self.prob)
# max pooling:
pool_1 = tf.nn.max_pool(drop_2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
return pool_1
def _bottleneck(inputs, channels, strides):
x = inputs
x = tf.keras.layers.Conv2D(channels, kernel_size=1, use_bias=False)(x)
x = GroupNormalization(channels // 4)(x)
x = tf.keras.layers.Activation(tf.keras.activations.relu)(x)
x = tf.keras.layers.Conv2D(channels,
kernel_size=3,
strides=strides,
use_bias=False,
padding="same")(x)
x = GroupNormalization(channels // 4)(x)
x = tf.keras.layers.Activation(tf.keras.activations.relu)(x)
x = tf.keras.layers.Conv2D(2 * channels, kernel_size=1, use_bias=False)(x)
x = GroupNormalization(channels // 4)(x)
if strides != 1 or inputs.shape[-1] != x.shape[-1]:
y = tf.keras.layers.Conv2D(2 * channels,
kernel_size=1,
strides=strides,
use_bias=False)(inputs)
res = GroupNormalization(channels // 4)(y)
x = tf.keras.layers.Add()([x, res])
return tf.keras.layers.Activation(tf.keras.activations.relu)(x)
def test_groupnorm_correctness_1d():
model = Sequential()
norm = GroupNormalization(input_shape=(10, ), groups=2)
model.add(norm)
model.compile(loss='mse', optimizer='rmsprop')
# centered on 5.0, variance 10.0
x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10))
model.fit(x, x, epochs=5, verbose=0)
out = model.predict(x)
out -= K.eval(norm.beta)
out /= K.eval(norm.gamma)
assert_allclose(out.mean(), 0.0, atol=1e-1)
assert_allclose(out.std(), 1.0, atol=1e-1)
def test_groupnorm_convnet():
model = Sequential()
norm = GroupNormalization(axis=1, input_shape=(3, 4, 4), groups=3)
model.add(norm)
model.compile(loss='mse', optimizer='sgd')
# centered on 5.0, variance 10.0
x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4))
model.fit(x, x, epochs=4, verbose=0)
out = model.predict(x)
out -= np.reshape(K.eval(norm.beta), (1, 3, 1, 1))
out /= np.reshape(K.eval(norm.gamma), (1, 3, 1, 1))
assert_allclose(np.mean(out, axis=(0, 2, 3)), 0.0, atol=1e-1)
assert_allclose(np.std(out, axis=(0, 2, 3)), 1.0, atol=1e-1)
def test_that_trainable_disables_updates():
val_a = np.random.random((10, 4))
val_out = np.random.random((10, 4))
a = Input(shape=(4, ))
layer = GroupNormalization(input_shape=(4, ), groups=2)
b = layer(a)
model = Model(a, b)
model.trainable = False
assert len(model.updates) == 0
model.compile('sgd', 'mse')
assert len(model.updates) == 0
x1 = model.predict(val_a)
model.train_on_batch(val_a, val_out)
x2 = model.predict(val_a)
assert_allclose(x1, x2, atol=1e-7)
model.trainable = True
model.compile('sgd', 'mse')
assert len(model.updates) == 0
model.train_on_batch(val_a, val_out)
x2 = model.predict(val_a)
assert np.abs(np.sum(x1 - x2)) > 1e-5
layer.trainable = False
model.compile('sgd', 'mse')
assert len(model.updates) == 0
x1 = model.predict(val_a)
model.train_on_batch(val_a, val_out)
x2 = model.predict(val_a)
assert_allclose(x1, x2, atol=1e-7)
def _construct_rgbflow_model(self):
special_init_done = False
for new_layer, weight_layer in zip(self.base_model.layers,
self.weights_model.layers):
# Find first conv layer
if isinstance(new_layer, Conv2D) and not special_init_done:
params = [x.copy() for x in weight_layer.get_weights()]
weight = params[0]
# RGB plus optical flow
H, W, C, num_filters = weight.shape
# Allocate new weights
new_kernel_size = (H, W, self.num_channels, num_filters)
new_kernels = np.zeros(new_kernel_size)
# See Motion Fused Frames paper, section 3.3 for initialization technique
new_kernels[:, :, :3, :] = weight
new_kernels[:, :, 3:, :] = np.tile(
np.mean(weight, axis=2, keepdims=True),
(1, 1, 2 * self.new_length, 1))
# Allocate new weights
new_kernel_size = (H, W, self.num_channels, num_filters)
new_kernels = np.zeros(new_kernel_size)
params[0] = new_kernels
new_layer.set_weights(params)
special_init_done = True
else:
# copy over downstream layers
new_layer.set_weights(weight_layer.get_weights())
# save input layer for model conversion
self.base_input_layer = self.base_model.input
if self.group_norm:
base_input_shape = (self.image_dim, self.image_dim,
self.num_channels)
gn_input = Input(shape=base_input_shape)
gn_tensor = GroupNormalization(groups=self.num_channels)(gn_input)
self.base_input_layer = gn_input
self.base_model = self.base_model(gn_tensor)
self.base_model = Model(gn_input, self.base_model)
# delete canibalized weights model
del self.weights_model
def test_groupnorm_convnet_no_center_no_scale():
model = Sequential()
norm = GroupNormalization(axis=-1,
center=False,
scale=False,
input_shape=(3, 4, 4),
groups=2)
model.add(norm)
model.compile(loss='mse', optimizer='sgd')
# centered on 5.0, variance 10.0
x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4))
model.fit(x, x, epochs=4, verbose=0)
out = model.predict(x)
assert_allclose(np.mean(out, axis=(0, 2, 3)), 0.0, atol=1e-1)
assert_allclose(np.std(out, axis=(0, 2, 3)), 1.0, atol=1e-1)
def edges_and_vertices(self):
"""
A model that predicts all vertices of an object within an image using
a similar architecture to the image encoder model.
input: RGB image of dimension 224x224x3
output: downsampled vertex predictions of dimension 28x28x1
"""
with tf.variable_scope("all_vertices", reuse=tf.AUTO_REUSE):
## boundary prediction:
conv_1 = tf.layers.conv2d(inputs=self.representation,
filters=32,
padding="same",
kernel_size=[3, 3],
name="conv_1")
gnorm_1 = GroupNormalization(groups=32, axis=-1,
epsilon=0.1)(conv_1)
relu_1 = tf.nn.relu(gnorm_1)
flat = tf.layers.Flatten()(relu_1)
E = tf.layers.dense(inputs=flat,
units=784,
activation=tf.nn.sigmoid)
E_plus_flat = tf.concat(
[flat, E],
axis=1) ## combine image representation and boundary output
V = tf.layers.dense(inputs=E_plus_flat,
units=784,
activation=tf.nn.sigmoid)
## get the index associated with the first vertex:
vertex_index = tf.argmax(V, axis=1)
## one-hot encode the first vertex:
vertex_encoding = tf.one_hot(indices=vertex_index, depth=784)
return tf.reshape(E, [-1, 28, 28, 1]), tf.reshape(
V, [-1, 28, 28, 1]), vertex_encoding
def green_block(inp, filters, data_format='channels_first', name=None):
"""
green_block(inp, filters, name=None)
------------------------------------
Implementation of the special residual block used in the paper. The block
consists of two (GroupNorm --> ReLu --> 3x3x3 non-strided Convolution)
units, with a residual connection from the input `inp` to the output. Used
internally in the model. Can be used independently as well.
Parameters
----------
`inp`: An keras.layers.layer instance, required
The keras layer just preceding the green block.
`filters`: integer, required
No. of filters to use in the 3D convolutional block. The output
layer of this green block will have this many no. of channels.
`data_format`: string, optional
The format of the input data. Must be either 'chanels_first' or
'channels_last'. Defaults to `channels_first`, as used in the paper.
`name`: string, optional
The name to be given to this green block. Defaults to None, in which
case, keras uses generated names for the involved layers. If a string
is provided, the names of individual layers are generated by attaching
a relevant prefix from [GroupNorm_, Res_, Conv3D_, Relu_, ], followed
by _1 or _2.
Returns
-------
`out`: A keras.layers.Layer instance
The output of the green block. Has no. of channels equal to `filters`.
The size of the rest of the dimensions remains same as in `inp`.
"""
inp_res = Conv3D(filters=filters,
kernel_size=(1, 1, 1),
strides=1,
data_format=data_format,
name=f'Res_{name}' if name else None)(inp)
# axis=1 for channels_first data format
# No. of groups = 8, as given in the paper
x = GroupNormalization(groups=8,
axis=1 if data_format == 'channels_first' else 0,
name=f'GroupNorm_1_{name}' if name else None)(inp)
x = Activation('relu', name=f'Relu_1_{name}' if name else None)(x)
x = Conv3D(filters=filters,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format=data_format,
name=f'Conv3D_1_{name}' if name else None)(x)
x = GroupNormalization(groups=8,
axis=1 if data_format == 'channels_first' else 0,
name=f'GroupNorm_2_{name}' if name else None)(x)
x = Activation('relu', name=f'Relu_2_{name}' if name else None)(x)
x = Conv3D(filters=filters,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format=data_format,
name=f'Conv3D_2_{name}' if name else None)(x)
out = Add(name=f'Out_{name}' if name else None)([x, inp_res])
return out
def build_model(input_shape=(4, 160, 192, 128),
output_channels=3,
weight_L2=0.1,
weight_KL=0.1,
learning_rate=1e-4):
"""
build_model(input_shape=(4, 160, 192, 128), output_channels=3, weight_L2=0.1, weight_KL=0.1)
-------------------------------------------
Creates the model used in the BRATS2018 winning solution
by Myronenko A. (https://arxiv.org/pdf/1810.11654.pdf)
Parameters
----------
`input_shape`: A 4-tuple, optional.
Shape of the input image. Must be a 4D image of shape (c, H, W, D),
where, each of H, W and D are divisible by 2^4, and c is divisible by 4.
Defaults to the crop size used in the paper, i.e., (4, 160, 192, 128).
`output_channels`: An integer, optional.
The no. of channels in the output. Defaults to 3 (BraTS 2018 format).
`weight_L2`: A real number, optional
The weight to be given to the L2 loss term in the loss function. Adjust to get best
results for your task. Defaults to 0.1.
`weight_KL`: A real number, optional
The weight to be given to the KL loss term in the loss function. Adjust to get best
results for your task. Defaults to 0.1.
Returns
-------
`model`: A keras.models.Model instance
The created model.
"""
c, H, W, D = input_shape
assert len(input_shape) == 4, "Input shape must be a 4-tuple"
assert (c % 4) == 0, "The no. of channels must be divisible by 4"
assert (H % 16) == 0 and (W % 16) == 0 and (D % 16) == 0, \
"All the input dimensions must be divisible by 16"
# -------------------------------------------------------------------------
# Encoder
# -------------------------------------------------------------------------
## Input Layer
inp = Input(input_shape)
## The Initial Block
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_x1')(inp)
## Dropout (0.2)
x = Dropout(0.2)(x)
## Green Block x1 (output filters = 32)
x1 = green_block(x, 32, name='x1')
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_32')(x1)
## Green Block x2 (output filters = 64)
x = green_block(x, 64, name='Enc_64_1')
x2 = green_block(x, 64, name='x2')
x = Conv3D(filters=64,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_64')(x2)
## Green Blocks x2 (output filters = 128)
x = green_block(x, 128, name='Enc_128_1')
x3 = green_block(x, 128, name='x3')
x = Conv3D(filters=128,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_128')(x3)
## Green Blocks x4 (output filters = 256)
x = green_block(x, 256, name='Enc_256_1')
x = green_block(x, 256, name='Enc_256_2')
x = green_block(x, 256, name='Enc_256_3')
x4 = green_block(x, 256, name='x4')
# -------------------------------------------------------------------------
# Decoder
# -------------------------------------------------------------------------
## GT (Groud Truth) Part
# -------------------------------------------------------------------------
### Green Block x1 (output filters=128)
x = Conv3D(filters=128,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_128')(x4)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_128')(x)
x = Add(name='Input_Dec_GT_128')([x, x3])
x = green_block(x, 128, name='Dec_GT_128')
### Green Block x1 (output filters=64)
x = Conv3D(filters=64,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_64')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_64')(x)
x = Add(name='Input_Dec_GT_64')([x, x2])
x = green_block(x, 64, name='Dec_GT_64')
### Green Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_32')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_32')(x)
x = Add(name='Input_Dec_GT_32')([x, x1])
x = green_block(x, 32, name='Dec_GT_32')
### Blue Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_Dec_GT_Output')(x)
x = Conv3D(
filters=output_channels, # No. of tumor classes is 3
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
activation='linear',
name='Dec_GT_Output')(x)
### Output Block
out_GT = Softmax(axis=1)(x)
## VAE (Variational Auto Encoder) Part
# -------------------------------------------------------------------------
### VD Block (Reducing dimensionality of the data)
x = GroupNormalization(groups=8, axis=1, name='Dec_VAE_VD_GN')(x4)
x = Activation('relu', name='Dec_VAE_VD_relu')(x)
x = Conv3D(filters=16,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Dec_VAE_VD_Conv3D')(x)
# Not mentioned in the paper, but the author used a Flattening layer here.
x = Flatten(name='Dec_VAE_VD_Flatten')(x)
x = Dense(256, name='Dec_VAE_VD_Dense')(x)
### VDraw Block (Sampling)
z_mean = Dense(128, name='Dec_VAE_VDraw_Mean')(x)
z_var = Dense(128, name='Dec_VAE_VDraw_Var')(x)
x = Lambda(sampling, name='Dec_VAE_VDraw_Sampling')([z_mean, z_var])
### VU Block (Upsizing back to a depth of 256)
# (10x12x8 neural)
x = Dense((c // 4) * (H // 16) * (W // 16) * (D // 16))(x)
x = Activation('relu')(x)
# (1x10x12x8)
x = Reshape(((c // 4), (H // 16), (W // 16), (D // 16)))(x)
x = Conv3D(filters=256,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_256')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_256')(x)
### Green Block x1 (output filters=128)
x = Conv3D(filters=128,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_128')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_128')(x)
x = green_block(x, 128, name='Dec_VAE_128')
### Green Block x1 (output filters=64)
x = Conv3D(filters=64,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_64')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_64')(x)
x = green_block(x, 64, name='Dec_VAE_64')
### Green Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_32')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_32')(x)
x = green_block(x, 32, name='Dec_VAE_32')
### Blue Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_Dec_VAE_Output')(x)
### Output Block
out_VAE = Conv3D(filters=4,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_Output')(x)
# Build and Compile the model
out = out_GT
model = Model(inp, out) # Create the model
model.compile(adam(lr=learning_rate),
loss(input_shape,
inp,
out_VAE,
z_mean,
z_var,
weight_L2=weight_L2,
weight_KL=weight_KL),
metrics=[dice_coefficient])
return model
def CAN_model(img_height_size, img_width_size, n_bands, num_classes, group_filters, rcm_4_filters = 512, cfm_gp_filters = 256,
cfm_conv_1_filters = 512, cfm_conv_2_filters = 256, amm_3_filters = 256, l_r = 0.0001):
"""
This function is used to generate the Context Aggregation Network (CAN) architecture as described in the paper 'Context
Aggregation Network for Semantic Labeling in Aerial Images' by Cheng W., Yang W., Wang M., Wang G., Chen J. (2019).
Inputs:
- img_height_size: Height of image patches to be used for model training
- img_width_size: Width of image patches to be used for model training
- n_bands: Number of channels contained in the image patches to be used for model training
- num_classes: Number of classes to be used for semantic segmentation
- group_filters: Number of groups to be used for group normalization
- rcm_4_filters: Number of filters to be used for Residual Convolutional Module (RCM) on Res - 4 output
- cfm_gp_filters: Number of filters to be used for global pooling component of Context Fuse Module (CFM)
- cfm_conv_1_filters: Number of filters to be used for first convolutions in parallel convolutional block of CFM
- cfm_conv_2_filters: Number of filters to be used for second convolutions in parallel convolutional block of CFM
- amm_3_filters: Number of filters to be used for convolutional layer in Attention Mix Module (AMM) on Res - 3 output
- l_r: Learning rate for the Adam optimizer
Outputs:
- can_model: CAN model to be trained using input parameters and network architecture
"""
rcm_3_filters = int(rcm_4_filters / 2)
rcm_2_filters = int(rcm_3_filters / 2)
rcm_1_filters = int(rcm_2_filters / 2)
rcm_base_filters = int(rcm_1_filters / 2)
amm_2_filters = int(amm_3_filters / 2)
amm_1_filters = int(amm_2_filters / 2)
resnet50_model = ResNet50(include_top = False, input_shape = (img_height_size, img_width_size, n_bands))
res_1 = resnet50_model.get_layer('activation_10').output
res_2 = resnet50_model.get_layer('activation_22').output
res_3 = resnet50_model.get_layer('activation_40').output
res_4 = resnet50_model.get_layer('activation_49').output
rcm_4_1_1_1 = Conv2D(rcm_4_filters, (1, 1), padding = 'same')(res_4)
rcm_4_1_3_3 = Conv2D(rcm_4_filters, (3, 3), padding = 'same')(rcm_4_1_1_1)
rcm_4_1_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_4_1_3_3)
rcm_4_1_3_3_act = Activation('relu')(rcm_4_1_3_3_gn)
rcm_4_1_2nd_3_3 = Conv2D(rcm_4_filters, (3, 3), padding = 'same')(rcm_4_1_3_3_act)
rcm_4_1_out = Add()([rcm_4_1_1_1, rcm_4_1_2nd_3_3])
cfm_b_gp = Lambda(lambda x: K.mean(x, axis = 3)[:, :, :, None])(rcm_4_1_out)
cfm_b_1_1 = Conv2D(cfm_gp_filters, (1, 1), padding = 'same')(cfm_b_gp)
cfm_b_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_b_1_1)
cfm_a_3_1 = Conv2D(cfm_conv_1_filters, (3, 3), padding = 'same')(rcm_4_1_out)
cfm_a_3_1_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_3_1)
cfm_a_3_1_act = Activation('relu')(cfm_a_3_1_gn)
cfm_a_3_2 = Conv2D(cfm_conv_2_filters, (3, 3), padding = 'same')(cfm_a_3_1_act)
cfm_a_3_2_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_3_2)
cfm_a_7_1 = Conv2D(cfm_conv_1_filters, (7, 7), padding = 'same')(rcm_4_1_out)
cfm_a_7_1_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_7_1)
cfm_a_7_1_act = Activation('relu')(cfm_a_7_1_gn)
cfm_a_7_2 = Conv2D(cfm_conv_2_filters, (7, 7), padding = 'same')(cfm_a_7_1_act)
cfm_a_7_2_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_7_2)
cfm_a_11_1 = Conv2D(cfm_conv_1_filters, (11, 11), padding = 'same')(rcm_4_1_out)
cfm_a_11_1_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_11_1)
cfm_a_11_1_act = Activation('relu')(cfm_a_11_1_gn)
cfm_a_11_2 = Conv2D(cfm_conv_2_filters, (11, 11), padding = 'same')(cfm_a_11_1_act)
cfm_a_11_2_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_11_2)
cfm_a_15_1 = Conv2D(cfm_conv_1_filters, (15, 15), padding = 'same')(rcm_4_1_out)
cfm_a_15_1_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_15_1)
cfm_a_15_1_act = Activation('relu')(cfm_a_15_1_gn)
cfm_a_15_2 = Conv2D(cfm_conv_2_filters, (15, 15), padding = 'same')(cfm_a_15_1_act)
cfm_a_15_2_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_15_2)
cfm_out = concatenate([cfm_b_gn, cfm_a_3_2_gn, cfm_a_7_2_gn, cfm_a_11_2_gn, cfm_a_15_2_gn])
rcm_4_2_1_1 = Conv2D(rcm_3_filters, (1, 1), padding = 'same')(cfm_out)
rcm_4_2_3_3 = Conv2D(rcm_3_filters, (3, 3), padding = 'same')(rcm_4_2_1_1)
rcm_4_2_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_4_2_3_3)
rcm_4_2_3_3_act = Activation('relu')(rcm_4_2_3_3_gn)
rcm_4_2_2nd_3_3 = Conv2D(rcm_3_filters, (3, 3), padding = 'same')(rcm_4_2_3_3_act)
rcm_4_2_out = Add()([rcm_4_2_1_1, rcm_4_2_2nd_3_3])
rcm_3_1_1_1 = Conv2D(rcm_3_filters, (1, 1), padding = 'same')(res_3)
rcm_3_1_3_3 = Conv2D(rcm_3_filters, (3, 3), padding = 'same')(rcm_3_1_1_1)
rcm_3_1_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_3_1_3_3)
rcm_3_1_3_3_act = Activation('relu')(rcm_3_1_3_3_gn)
rcm_3_1_2nd_3_3 = Conv2D(rcm_3_filters, (3, 3), padding = 'same')(rcm_3_1_3_3_act)
rcm_3_1_out = Add()([rcm_3_1_1_1, rcm_3_1_2nd_3_3])
upsam_rcm_4 = UpSampling2D(size = (2, 2), interpolation = 'bilinear')(rcm_4_2_out)
amm_3_input = concatenate([rcm_3_1_out, upsam_rcm_4])
amm_3_conv = Conv2D(amm_3_filters, (3, 3), padding = 'same')(amm_3_input)
amm_3_conv_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(amm_3_conv)
amm_3_conv_act = Activation('relu')(amm_3_conv_gn)
amm_3_gp = Lambda(lambda x: K.mean(x, axis = 3)[:, :, :, None])(amm_3_conv_act)
amm_3_mult = Multiply()([amm_3_conv_act, amm_3_gp])
amm_3_out = Add()([amm_3_mult, rcm_3_1_out])
rcm_3_2_1_1 = Conv2D(rcm_2_filters, (1, 1), padding = 'same')(amm_3_out)
rcm_3_2_3_3 = Conv2D(rcm_2_filters, (3, 3), padding = 'same')(rcm_3_2_1_1)
rcm_3_2_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_3_2_3_3)
rcm_3_2_3_3_act = Activation('relu')(rcm_3_2_3_3_gn)
rcm_3_2_2nd_3_3 = Conv2D(rcm_2_filters, (3, 3), padding = 'same')(rcm_3_2_3_3_act)
rcm_3_2_out = Add()([rcm_3_2_1_1, rcm_3_2_2nd_3_3])
rcm_2_1_1_1 = Conv2D(rcm_2_filters, (1, 1), padding = 'same')(res_2)
rcm_2_1_3_3 = Conv2D(rcm_2_filters, (3, 3), padding = 'same')(rcm_2_1_1_1)
rcm_2_1_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_2_1_3_3)
rcm_2_1_3_3_act = Activation('relu')(rcm_2_1_3_3_gn)
rcm_2_1_2nd_3_3 = Conv2D(rcm_2_filters, (3, 3), padding = 'same')(rcm_2_1_3_3_act)
rcm_2_1_out = Add()([rcm_2_1_1_1, rcm_2_1_2nd_3_3])
upsam_rcm_3 = UpSampling2D(size = (2, 2), interpolation = 'bilinear')(rcm_3_2_out)
amm_2_input = concatenate([rcm_2_1_out, upsam_rcm_3])
amm_2_conv = Conv2D(amm_2_filters, (3, 3), padding = 'same')(amm_2_input)
amm_2_conv_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(amm_2_conv)
amm_2_conv_act = Activation('relu')(amm_2_conv_gn)
amm_2_gp = Lambda(lambda x: K.mean(x, axis = 3)[:, :, :, None])(amm_2_conv_act)
amm_2_mult = Multiply()([amm_2_conv_act, amm_2_gp])
amm_2_out = Add()([amm_2_mult, rcm_2_1_out])
rcm_2_2_1_1 = Conv2D(rcm_1_filters, (1, 1), padding = 'same')(amm_2_out)
rcm_2_2_3_3 = Conv2D(rcm_1_filters, (3, 3), padding = 'same')(rcm_2_2_1_1)
rcm_2_2_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_2_2_3_3)
rcm_2_2_3_3_act = Activation('relu')(rcm_2_2_3_3_gn)
rcm_2_2_2nd_3_3 = Conv2D(rcm_1_filters, (3, 3), padding = 'same')(rcm_2_2_3_3_act)
rcm_2_2_out = Add()([rcm_2_2_1_1, rcm_2_2_2nd_3_3])
rcm_1_1_1_1 = Conv2D(rcm_1_filters, (1, 1), padding = 'same')(res_1)
rcm_1_1_3_3 = Conv2D(rcm_1_filters, (3, 3), padding = 'same')(rcm_1_1_1_1)
rcm_1_1_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_1_1_3_3)
rcm_1_1_3_3_act = Activation('relu')(rcm_1_1_3_3_gn)
rcm_1_1_2nd_3_3 = Conv2D(rcm_1_filters, (3, 3), padding = 'same')(rcm_1_1_3_3_act)
rcm_1_1_out = Add()([rcm_1_1_1_1, rcm_1_1_2nd_3_3])
upsam_rcm_2 = UpSampling2D(size = (2, 2), interpolation = 'bilinear')(rcm_2_2_out)
amm_1_input = concatenate([rcm_1_1_out, upsam_rcm_2])
amm_1_conv = Conv2D(amm_1_filters, (3, 3), padding = 'same')(amm_1_input)
amm_1_conv_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(amm_1_conv)
amm_1_conv_act = Activation('relu')(amm_1_conv_gn)
amm_1_gp = Lambda(lambda x: K.mean(x, axis = 3)[:, :, :, None])(amm_1_conv_act)
amm_1_mult = Multiply()([amm_1_conv_act, amm_1_gp])
amm_1_out = Add()([amm_1_mult, rcm_1_1_out])
rcm_1_2_1_1 = Conv2D(rcm_base_filters, (1, 1), padding = 'same')(amm_1_out)
rcm_1_2_3_3 = Conv2D(rcm_base_filters, (3, 3), padding = 'same')(rcm_1_2_1_1)
rcm_1_2_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_1_2_3_3)
rcm_1_2_3_3_act = Activation('relu')(rcm_1_2_3_3_gn)
rcm_1_2_2nd_3_3 = Conv2D(rcm_base_filters, (3, 3), padding = 'same')(rcm_1_2_3_3_act)
rcm_1_2_out = Add()([rcm_1_2_1_1, rcm_1_2_2nd_3_3])
deconv_2 = Conv2DTranspose(rcm_base_filters, (2, 2), strides = (2, 2), padding = 'same')(rcm_1_2_out)
deconv_2_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(deconv_2)
deconv_2_act = Activation('relu')(deconv_2_gn)
deconv_1 = Conv2DTranspose(rcm_base_filters, (2, 2), strides = (2, 2), padding = 'same')(deconv_2_act)
deconv_1_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(deconv_1)
deconv_1_act = Activation('relu')(deconv_1_gn)
pred_layer = Conv2D(num_classes, (1, 1), padding = 'same', activation = 'softmax')(deconv_1_act)
can_model = Model(inputs = resnet50_model.input, outputs = pred_layer)
can_model.compile(loss = 'categorical_crossentropy', optimizer = Adam(lr = l_r), metrics = ['categorical_crossentropy'])
return can_model
def create_model(dim=(256, 256), weights=np.ones(5), split=False):
f_loss = categorical_focal_loss(alpha=weights)
IMG_WIDTH, IMG_HEIGHT, CHANNELS = *dim, 3
input_shape = (IMG_WIDTH, IMG_HEIGHT, CHANNELS)
elu = keras.layers.ELU(alpha=1.0)
# create the base pre-trained model
# Load in EfficientNetB5
effnet = efn.EfficientNetB4(weights=None,
include_top=False,
input_shape=(IMG_WIDTH, IMG_HEIGHT, CHANNELS))
effnet.load_weights(
'/media/parth/DATA/datasets/aptos_2019/efficientnet-b4_imagenet_1000_notop.h5'
)
# Replace all Batch Normalization layers by Group Normalization layers
for i, layer in enumerate(effnet.layers):
if "batch_normalization" in layer.name:
effnet.layers[i] = GroupNormalization(groups=32,
axis=-1,
epsilon=0.00001)
if split == True:
input1 = Input(input_shape)
input2 = Input(input_shape)
input3 = Input(input_shape)
input4 = Input(input_shape)
conv1 = Conv2D(16, 3, padding='same')(input1)
conv2 = Conv2D(16, 3, padding='same')(input2)
conv3 = Conv2D(16, 3, padding='same')(input3)
conv4 = Conv2D(16, 3, padding='same')(input4)
concat = concatenate([conv1, conv2, conv3, conv4])
enet_input = Conv2D(3, 3, padding='same')(concat)
x = effnet(enet_input)
x = GlobalAveragePooling2D()(x)
x = Dense(256)(x)
x = Dropout(0.25)(x)
x = Dense(5)(x)
predictions = Softmax()(x)
model = Model(inputs=[input1, input2, input3, input4],
outputs=predictions)
model.compile(loss=f_loss,
optimizer=RAdam(learning_rate=0.00005),
metrics=[f_loss, 'acc'])
print(model.summary())
return model
else:
x = effnet.output
x = GlobalAveragePooling2D()(x)
x = Dense(256)(x)
x = Dropout(0.25)(x)
x = Dense(5)(x)
predictions = Softmax()(x)
model = Model(inputs=effnet.input, outputs=predictions)
model.compile(loss=f_loss,
optimizer=RAdam(lr=0.00005),
metrics=[f_loss, 'acc'])
print(model.summary())
return model
print('Pad sequences (samples x time)')
x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
x_val = sequence.pad_sequences(x_val, maxlen=maxlen)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)
print('x_val shape:', x_val.shape)
# configuration matches 4.47 Million parameters with `units=600` and `64 embedding dim`
print('Build model...')
inputs = Input(shape=(maxlen, ))
embed = Embedding(embed_size + 1, 128, input_shape=(maxlen, ))(inputs)
groupNormal = GroupNormalization(groups=32, axis=-1)(embed)
x_score = Convolution1D(filters=1,
kernel_size=3,
padding='same',
activation='sigmoid')(groupNormal)
x_atten = Multiply()([x_score, embed])
first_ind = IndRNN(FLAGS.units,
recurrent_clip_min=-1,
recurrent_clip_max=-1,
dropout=0.0,
recurrent_dropout=0.0,
return_sequences=True)(x_atten)
second_ind = IndRNN(FLAGS.units,
recurrent_clip_min=-1,
recurrent_clip_max=-1,
dropout=0.0,
def JointNet(img_height_size = 512, img_width_size = 512, n_bands = 3, growth_rate_1 = 32, growth_rate_2 = 64,
growth_rate_3 = 128, growth_rate_net_bridge = 256, group_filters = 8, l_r = 0.0001):
"""
This function is used to generate the JointNet architecture as described in the paper 'JointNet: A Common Neural Network
for Road and Building Extraction' by Zhang Z., Wang Y. (2019).
Inputs:
- img_height_size: Height of image patches to be used for model training
- img_width_size: Width of image patches to be used for model training
- n_bands: Number of channels contained in the image patches to be used for model training
- growth_rate_1: Number of filters to be used for each atrous convolution in the first dense atrous convolution block
- growth_rate_2: Number of filters to be used for each atrous convolution in the second dense atrous convolution block
- growth_rate_3: Number of filters to be used for each atrous convolution in the third dense atrous convolution block
- growth_rate_net_bridge: Number of filters to be used for each atrous convolution in the network bridge
- group_filters: Number of groups to be used for group normalization
- l_r: Learning rate to be applied for the Adam optimizer
Outputs:
- jointnet_model: JointNet model to be trained using input parameters and network architecture
"""
img_input = Input(shape = (img_height_size, img_width_size, n_bands))
ec_1_ac_1 = Conv2D(growth_rate_1, (3, 3), padding = 'same', activation = 'relu')(img_input)
ec_1_ac_1_gn = GroupNormalization(groups = group_filters, axis = -1,
epsilon = 0.1)(ec_1_ac_1)
ec_1_ac_1_output = concatenate([img_input, ec_1_ac_1_gn])
ec_1_ac_2 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(ec_1_ac_1_output)
ec_1_ac_2_gn = GroupNormalization(groups = group_filters, axis = -1,
epsilon = 0.1)(ec_1_ac_2)
ec_1_ac_2_output = concatenate([img_input, ec_1_ac_1_gn, ec_1_ac_2_gn])
ec_1_ac_3 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(ec_1_ac_2_output)
ec_1_ac_3_gn = GroupNormalization(groups = group_filters, axis = -1,
epsilon = 0.1)(ec_1_ac_3)
ec_1_ac_3_output = concatenate([img_input, ec_1_ac_1_gn, ec_1_ac_2_gn, ec_1_ac_3_gn])
ec_1_ac_4 = Conv2D(growth_rate_1, (3, 3), padding = 'same', activation = 'relu')(ec_1_ac_3_output)
ec_1_ac_4_gn = GroupNormalization(groups = group_filters, axis = -1,
epsilon = 0.1)(ec_1_ac_4)
ec_1_ac_4_output = concatenate([img_input, ec_1_ac_1_gn, ec_1_ac_2_gn, ec_1_ac_3_gn, ec_1_ac_4_gn])
ec_1_ac_5 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(ec_1_ac_4_output)
ec_1_ac_5_gn = GroupNormalization(groups = group_filters, axis = -1,
epsilon = 0.1)(ec_1_ac_5)
ec_1_ac_5_output = concatenate([img_input, ec_1_ac_1_gn, ec_1_ac_2_gn, ec_1_ac_3_gn, ec_1_ac_4_gn, ec_1_ac_5_gn])
ec_1_ac_6 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(ec_1_ac_5_output)
ec_1_ac_6_gn = GroupNormalization(groups = group_filters, axis = -1,
epsilon = 0.1)(ec_1_ac_6)
ec_1_ac_6_output = concatenate([img_input, ec_1_ac_1_gn, ec_1_ac_2_gn, ec_1_ac_3_gn, ec_1_ac_4_gn, ec_1_ac_5_gn,
ec_1_ac_6_gn])
ec_1_DO = Conv2D(int(4 * growth_rate_1), (1, 1), padding = 'same', activation = 'relu')(ec_1_ac_6_output)
ec_1_RO = Conv2D(growth_rate_1, (1, 1), strides = (2, 2), padding = 'same', activation = 'relu')(img_input)
ec_2_ac_1 = Conv2D(growth_rate_2, (3, 3), padding = 'same', activation = 'relu')(ec_1_RO)
ec_2_ac_1_gn = GroupNormalization(groups = int(group_filters * 2), axis = -1,
epsilon = 0.1)(ec_2_ac_1)
ec_2_ac_1_output = concatenate([ec_1_RO, ec_2_ac_1_gn])
ec_2_ac_2 = Conv2D(growth_rate_2, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(ec_2_ac_1_output)
ec_2_ac_2_gn = GroupNormalization(groups = int(group_filters * 2), axis = -1,
epsilon = 0.1)(ec_2_ac_2)
ec_2_ac_2_output = concatenate([ec_1_RO, ec_2_ac_1_gn, ec_2_ac_2_gn])
ec_2_ac_3 = Conv2D(growth_rate_2, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(ec_2_ac_2_output)
ec_2_ac_3_gn = GroupNormalization(groups = int(group_filters * 2), axis = -1,
epsilon = 0.1)(ec_2_ac_3)
ec_2_ac_3_output = concatenate([ec_1_RO, ec_2_ac_1_gn, ec_2_ac_2_gn, ec_2_ac_3_gn])
ec_2_ac_4 = Conv2D(growth_rate_2, (3, 3), padding = 'same', activation = 'relu')(ec_2_ac_3_output)
ec_2_ac_4_gn = GroupNormalization(groups = int(group_filters * 2), axis = -1,
epsilon = 0.1)(ec_2_ac_4)
ec_2_ac_4_output = concatenate([ec_1_RO, ec_2_ac_1_gn, ec_2_ac_2_gn, ec_2_ac_3_gn, ec_2_ac_4_gn])
ec_2_ac_5 = Conv2D(growth_rate_2, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(ec_2_ac_4_output)
ec_2_ac_5_gn = GroupNormalization(groups = int(group_filters * 2), axis = -1,
epsilon = 0.1)(ec_2_ac_5)
ec_2_ac_5_output = concatenate([ec_1_RO, ec_2_ac_1_gn, ec_2_ac_2_gn, ec_2_ac_3_gn, ec_2_ac_4_gn, ec_2_ac_5_gn])
ec_2_ac_6 = Conv2D(growth_rate_2, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(ec_2_ac_5_output)
ec_2_ac_6_gn = GroupNormalization(groups = int(group_filters * 2), axis = -1,
epsilon = 0.1)(ec_2_ac_6)
ec_2_ac_6_output = concatenate([ec_1_RO, ec_2_ac_1_gn, ec_2_ac_2_gn, ec_2_ac_3_gn, ec_2_ac_4_gn, ec_2_ac_5_gn,
ec_2_ac_6_gn])
ec_2_DO = Conv2D(int(4 * growth_rate_2), (1, 1), padding = 'same', activation = 'relu')(ec_2_ac_6_output)
ec_2_RO = Conv2D(growth_rate_2, (1, 1), strides = (2, 2), padding = 'same', activation = 'relu')(ec_1_RO)
ec_3_ac_1 = Conv2D(growth_rate_3, (3, 3), padding = 'same', activation = 'relu')(ec_2_RO)
ec_3_ac_1_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1,
epsilon = 0.1)(ec_3_ac_1)
ec_3_ac_1_output = concatenate([ec_2_RO, ec_3_ac_1_gn])
ec_3_ac_2 = Conv2D(growth_rate_3, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(ec_3_ac_1_output)
ec_3_ac_2_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1,
epsilon = 0.1)(ec_3_ac_2)
ec_3_ac_2_output = concatenate([ec_2_RO, ec_3_ac_1_gn, ec_3_ac_2_gn])
ec_3_ac_3 = Conv2D(growth_rate_3, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(ec_3_ac_2_output)
ec_3_ac_3_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1,
epsilon = 0.1)(ec_3_ac_3)
ec_3_ac_3_output = concatenate([ec_2_RO, ec_3_ac_1_gn, ec_3_ac_2_gn, ec_3_ac_3_gn])
ec_3_ac_4 = Conv2D(growth_rate_3, (3, 3), padding = 'same', activation = 'relu')(ec_3_ac_3_output)
ec_3_ac_4_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1,
epsilon = 0.1)(ec_3_ac_4)
ec_3_ac_4_output = concatenate([ec_2_RO, ec_3_ac_1_gn, ec_3_ac_2_gn, ec_3_ac_3_gn, ec_3_ac_4_gn])
ec_3_ac_5 = Conv2D(growth_rate_3, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(ec_3_ac_4_output)
ec_3_ac_5_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1,
epsilon = 0.1)(ec_3_ac_5)
ec_3_ac_5_output = concatenate([ec_2_RO, ec_3_ac_1_gn, ec_3_ac_2_gn, ec_3_ac_3_gn, ec_3_ac_4_gn, ec_3_ac_5_gn])
ec_3_ac_6 = Conv2D(growth_rate_3, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(ec_3_ac_5_output)
ec_3_ac_6_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1,
epsilon = 0.1)(ec_3_ac_6)
ec_3_ac_6_output = concatenate([ec_2_RO, ec_3_ac_1_gn, ec_3_ac_2_gn, ec_3_ac_3_gn, ec_3_ac_4_gn, ec_3_ac_5_gn,
ec_3_ac_6_gn])
ec_3_DO = Conv2D(int(4 * growth_rate_3), (1, 1), padding = 'same', activation = 'relu')(ec_3_ac_6_output)
ec_3_RO = Conv2D(growth_rate_3, (1, 1), strides = (2, 2), padding = 'same', activation = 'relu')(ec_2_RO)
net_bridge = Conv2D(growth_rate_net_bridge, (1, 1), padding = 'same', activation = 'relu')(ec_3_RO)
net_bridge_up = UpSampling2D(size = (2, 2), interpolation = 'bilinear')(net_bridge)
dc_3_input = concatenate([net_bridge_up, ec_3_DO])
dc_3_output = Conv2D(growth_rate_3, (1, 1), padding = 'same', activation = 'relu')(dc_3_input)
dc_3_up = UpSampling2D(size = (2, 2), interpolation = 'bilinear')(dc_3_output)
dc_2_input = concatenate([dc_3_up, ec_2_DO])
dc_2_output = Conv2D(growth_rate_2, (1, 1), padding = 'same', activation = 'relu')(dc_2_input)
dc_2_up = UpSampling2D(size = (2, 2), interpolation = 'bilinear')(dc_2_output)
dc_1_input = concatenate([dc_2_up, ec_1_DO])
dc_1_ac_1 = Conv2D(growth_rate_1, (3, 3), padding = 'same', activation = 'relu')(dc_1_input)
dc_1_ac_1_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1,
epsilon = 0.1)(dc_1_ac_1)
dc_1_ac_1_output = concatenate([dc_1_input, dc_1_ac_1_gn])
dc_1_ac_2 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(dc_1_ac_1_output)
dc_1_ac_2_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1,
epsilon = 0.1)(dc_1_ac_2)
dc_1_ac_2_output = concatenate([dc_1_input, dc_1_ac_1_gn, dc_1_ac_2_gn])
dc_1_ac_3 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(dc_1_ac_2_output)
dc_1_ac_3_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1,
epsilon = 0.1)(dc_1_ac_3)
dc_1_ac_3_output = concatenate([dc_1_input, dc_1_ac_1_gn, dc_1_ac_2_gn, dc_1_ac_3_gn])
dc_1_ac_4 = Conv2D(growth_rate_1, (3, 3), padding = 'same', activation = 'relu')(dc_1_ac_3_output)
dc_1_ac_4_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1,
epsilon = 0.1)(dc_1_ac_4)
dc_1_ac_4_output = concatenate([dc_1_input, dc_1_ac_1_gn, dc_1_ac_2_gn, dc_1_ac_3_gn, dc_1_ac_4_gn])
dc_1_ac_5 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(dc_1_ac_4_output)
dc_1_ac_5_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1,
epsilon = 0.1)(dc_1_ac_5)
dc_1_ac_5_output = concatenate([dc_1_input, dc_1_ac_1_gn, dc_1_ac_2_gn, dc_1_ac_3_gn, dc_1_ac_4_gn, dc_1_ac_5_gn])
dc_1_ac_6 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(dc_1_ac_5_output)
dc_1_ac_6_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1,
epsilon = 0.1)(dc_1_ac_6)
dc_1_ac_6_output = concatenate([dc_1_input, dc_1_ac_1_gn, dc_1_ac_2_gn, dc_1_ac_3_gn, dc_1_ac_4_gn, dc_1_ac_5_gn,
dc_1_ac_6_gn])
dc_1_DO = Conv2D(int(4 * growth_rate_1), (1, 1), padding = 'same', activation = 'relu')(dc_1_ac_6_output)
class_layer = Conv2D(1, (1, 1), padding = 'same', activation = 'sigmoid')(dc_1_DO)
jointnet_model = Model(inputs = img_input, outputs = class_layer)
jointnet_model.compile(loss = dice_coef_loss, optimizer = Adam(lr = l_r), metrics = [dice_coef])
return jointnet_model
def global_context_block(ip,
reduction_ratio=16,
transform_activation='linear'):
"""
Adds a Global Context attention block for self attention to the input tensor.
Input tensor can be or rank 3 (temporal), 4 (spatial) or 5 (spatio-temporal).
# Arguments:
ip: input tensor
intermediate_dim: The dimension of the intermediate representation. Can be
`None` or a positive integer greater than 0. If `None`, computes the
intermediate dimension as half of the input channel dimension.
reduction_ratio: Reduces the input filters by this factor for the
bottleneck block of the transform submodule. Node: the reduction
ratio must be set such that it divides the input number of channels,
transform_activation: activation function to apply to the output
of the transform block. Can be any string activation function availahle
to Keras.
# Returns:
a tensor of same shape as input
"""
channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
ip_shape = K.int_shape(ip)
# check rank and calculate the input shape
if len(ip_shape) == 3: # temporal / time series data
rank = 3
batchsize, dim1, channels = ip_shape
elif len(ip_shape) == 4: # spatial / image data
rank = 4
if channel_dim == 1:
batchsize, channels, dim1, dim2 = ip_shape
else:
batchsize, dim1, dim2, channels = ip_shape
elif len(ip_shape) == 5: # spatio-temporal / Video or Voxel data
rank = 5
if channel_dim == 1:
batchsize, channels, dim1, dim2, dim3 = ip_shape
else:
batchsize, dim1, dim2, dim3, channels = ip_shape
else:
raise ValueError(
'Input dimension has to be either 3 (temporal), 4 (spatial) or 5 (spatio-temporal)'
)
if rank > 3:
flat_spatial_dim = -1 if K.image_data_format(
) == 'channels_first' else 1
else:
flat_spatial_dim = 1
""" Context Modelling Block """
# [B, ***, C] or [B, C, ***]
input_flat = _spatial_flattenND(ip, rank)
# [B, ..., C] or [B, C, ...]
context = _convND(ip, rank, channels=1, kernel=1)
# [B, ..., 1] or [B, 1, ...]
context = _spatial_flattenND(context, rank)
# [B, ***, 1] or [B, 1, ***]
context = Softmax(axis=flat_spatial_dim)(context)
# Compute context block outputs
context = dot([input_flat, context], axes=flat_spatial_dim)
# [B, C, 1]
context = _spatial_expandND(context, rank)
# [B, C, 1...] or [B, 1..., C]
""" Transform block """
# Transform bottleneck
# [B, C // R, 1...] or [B, 1..., C // R]
transform = _convND(context, rank, channels // reduction_ratio, kernel=1)
# Group normalization acts as Layer Normalization when groups = 1
transform = GroupNormalization(groups=1, axis=channel_dim)(transform)
transform = Activation('relu')(transform)
# Transform output block
# [B, C, 1...] or [B, 1..., C]
transform = _convND(transform, rank, channels, kernel=1)
transform = Activation(transform_activation)(transform)
# apply context transform
out = add([ip, transform])
return out
def build_model(input_shape=(4, 160, 192, 128)):
"""
build_model(input_shape=(4, 160, 192, 128))
-------------------------------------------
Creates the model used in the BRATS2018 winning solution
by Myronenko A. (https://arxiv.org/pdf/1810.11654.pdf)
Parameters
----------
`input_shape`: A 4-tuple, optional.
Shape of the input image. Must be a 4D image of shape (c, H, W, D),
where, each of H, W and D are divisible by 2^4. Defaults to the crop
size used in the paper, i.e., (4, 160, 192, 128).
Returns
-------
`model`: A keras.models.Model instance
The created model.
"""
c, H, W, D = input_shape
assert len(input_shape) == 4, "Input shape must be a 4-tuple"
assert ~(H % 16) and ~(W % 16) and ~(D % 16), \
"All the input dimensions must be divisible by 16"
# -------------------------------------------------------------------------
# Encoder
# -------------------------------------------------------------------------
## Input Layer
inp = Input(input_shape)
## The Initial Block
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_x1')(inp)
## Dropout (0.2)
x = Dropout(0.2)(x)
## Green Block x1 (output filters = 32)
x1 = green_block(x, 32, name='x1')
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_32')(x1)
## Green Block x2 (output filters = 64)
x = green_block(x, 64, name='Enc_64_1')
x2 = green_block(x, 64, name='x2')
x = Conv3D(filters=64,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_64')(x2)
## Green Blocks x2 (output filters = 128)
x = green_block(x, 128, name='Enc_128_1')
x3 = green_block(x, 128, name='x3')
x = Conv3D(filters=128,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_128')(x3)
## Green Blocks x4 (output filters = 256)
x = green_block(x, 256, name='Enc_256_1')
x = green_block(x, 256, name='Enc_256_2')
x = green_block(x, 256, name='Enc_256_3')
x4 = green_block(x, 256, name='x4')
# -------------------------------------------------------------------------
# Decoder
# -------------------------------------------------------------------------
## GT (Groud Truth) Part
# -------------------------------------------------------------------------
### Green Block x1 (output filters=128)
x = Conv3D(filters=128,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_128')(x4)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_128')(x)
x = Add(name='Input_Dec_GT_128')([x, x3])
x = green_block(x, 128, name='Dec_GT_128')
### Green Block x1 (output filters=64)
x = Conv3D(filters=64,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_64')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_64')(x)
x = Add(name='Input_Dec_GT_64')([x, x2])
x = green_block(x, 64, name='Dec_GT_64')
### Green Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_32')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_32')(x)
x = Add(name='Input_Dec_GT_32')([x, x1])
x = green_block(x, 32, name='Dec_GT_32')
### Blue Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_Dec_GT_Output')(x)
### Output Block
out_GT = Conv3D(
filters=3, # No. of tumor classes is 3
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
activation='sigmoid',
name='Dec_GT_Output')(x)
## VAE (Variational Auto Encoder) Part
# -------------------------------------------------------------------------
### VD Block (Reducing dimensionality of the data)
x = GroupNormalization(groups=8, axis=1, name='Dec_VAE_VD_GN')(x4)
x = Activation('relu', name='Dec_VAE_VD_relu')(x)
x = Conv3D(filters=16,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Dec_VAE_VD_Conv3D')(x)
# Not mentioned in the paper, but the author used a Flattening layer here.
x = Flatten(name='Dec_VAE_VD_Flatten')(x)
x = Dense(256, name='Dec_VAE_VD_Dense')(x)
### VDraw Block (Sampling)
z_mean = Dense(128, name='Dec_VAE_VDraw_Mean')(x)
z_var = Dense(128, name='Dec_VAE_VDraw_Var')(x)
x = Lambda(sampling, name='Dec_VAE_VDraw_Sampling')([z_mean, z_var])
### VU Block (Upsizing back to a depth of 256)
x = Dense(1 * 10 * 12 * 8)(x)
x = Activation('relu')(x)
x = Reshape((1, 10, 12, 8))(x)
x = Conv3D(filters=256,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_256')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_256')(x)
### Green Block x1 (output filters=128)
x = Conv3D(filters=128,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_128')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_128')(x)
x = green_block(x, 128, name='Dec_VAE_128')
### Green Block x1 (output filters=64)
x = Conv3D(filters=64,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_64')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_64')(x)
x = green_block(x, 64, name='Dec_VAE_64')
### Green Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_32')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_32')(x)
x = green_block(x, 32, name='Dec_VAE_32')
### Blue Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_Dec_VAE_Output')(x)
### Output Block
out_VAE = Conv3D(filters=4,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_Output')(x)
# Build and Compile the model
out = out_GT
model = Model(inp, out) # Create the model
model.compile(adam(lr=1e-4),
loss(input_shape, inp, out_VAE, z_mean, z_var),
metrics=['accuracy'])
return model
