def separable_resnet_block(n_filters, input, dilation_rate=(1, 1)): """ Puts together a ResNet block with two _separable_ convolutions. :param n_filters: Number of filters ("depth") of this convolution :param input: Keras functional object representing input to this resnet block. :param dilation_rate: Tuple of 2 integers, specifying the dilation rate for dilated convolution. :return: Tensor, representing the output of this block. """ n_in = GroupNormalization(groups=n_filters // 4)(input) a_in = Activation(tf.nn.leaky_relu)(n_in) c1 = SeparableConv2D(filters=n_filters, kernel_size=(3, 3), strides=(1, 1), dilation_rate=dilation_rate, padding='same', use_bias=False)(a_in) n1 = GroupNormalization(groups=n_filters // 4)(c1) a1 = Activation(tf.nn.leaky_relu)(n1) c2 = SeparableConv2D(filters=n_filters, kernel_size=(3, 3), strides=(1, 1), dilation_rate=dilation_rate, padding='same', use_bias=False)(a1) s = Add()([input, c2]) return s
def resnet_block(n_filters, input, dilation_rate=(1, 1)): """ Pre-activation ResNet block with group normalization. :param n_filters: Number of filters ("depth") of this convolution :param input: Keras functional object representing input to this resnet block. :return: Keras functional object representing output of this resnet block. """ n_in = GroupNormalization(groups=n_filters // 4)(input) a_in = Activation(tf.nn.leaky_relu)(n_in) c1 = Conv2D(filters=n_filters, kernel_size=(3, 3), strides=(1, 1), dilation_rate=dilation_rate, padding='same', use_bias=False)(a_in) n1 = GroupNormalization(groups=n_filters // 4)(c1) a1 = Activation(tf.nn.leaky_relu)(n1) c2 = Conv2D(filters=n_filters, kernel_size=(3, 3), strides=(1, 1), dilation_rate=dilation_rate, padding='same', use_bias=False)(a1) s = Add()([input, c2]) return s
def getModel(shape, loss='mse', optimizer='adam'): # https://devblogs.nvidia.com/deep-learning-self-driving-cars/ NModel = Sequential() # Normalization Layer NModel.add(Lambda(lambda x: (x / 127.5) - 1., input_shape=shape)) # Convolutional Layer 1 NModel.add(Conv2D(filters=24, kernel_size=5, strides=(2, 2))) # Group normalization module for keras. # Downloaded from https://github.com/titu1994/Keras-Group-Normalization NModel.add(GroupNormalization(groups=12, axis=-1)) NModel.add(Activation('relu')) # Convolutional Layer 2 NModel.add(Conv2D(filters=36, kernel_size=5, strides=(2, 2))) NModel.add(GroupNormalization(groups=18, axis=-1)) NModel.add(Activation('relu')) # Convolutional Layer 3 NModel.add(Conv2D(filters=48, kernel_size=5, strides=(2, 2))) NModel.add(GroupNormalization(groups=12, axis=-1)) NModel.add(Activation('relu')) # Convolutional Layer 4 NModel.add(Conv2D(filters=64, kernel_size=3, strides=(1, 1))) NModel.add(GroupNormalization(groups=32, axis=-1)) NModel.add(Activation('relu')) # Convolutional Layer 5 NModel.add(Conv2D(filters=64, kernel_size=3, strides=(1, 1))) NModel.add(GroupNormalization(groups=32, axis=-1)) NModel.add(Activation('relu')) # Flatten Layers NModel.add(Flatten()) # Fully Connected Layer 1 NModel.add(Dense(100)) NModel.add(Activation('relu')) # Fully Connected Layer 2 NModel.add(Dense(50)) NModel.add(Activation('relu')) # Fully Connected Layer 3 NModel.add(Dense(10)) NModel.add(Activation('relu')) # Output Layer NModel.add(Dense(1)) NModel.compile(loss=loss, optimizer=optimizer) return NModel
def test_groupnorm_mode_twice(): # This is a regression test for issue #4881 with the old # batch normalization functions in the Theano backend. model = Sequential() model.add(GroupNormalization(input_shape=(10, 5, 5), axis=1, groups=2)) model.add(GroupNormalization(input_shape=(10, 5, 5), axis=1, groups=2)) model.compile(loss='mse', optimizer='sgd') x = np.random.normal(loc=5.0, scale=10.0, size=(20, 10, 5, 5)) model.fit(x, x, epochs=1, verbose=0) model.predict(x)
def block_5(self): """ Fifth block of VGG-16. input: feature map of dimension 16x16x512 output: feature map of dimension 8x8x512 """ with tf.variable_scope("block_5", reuse=tf.AUTO_REUSE): tf.set_random_seed(self.seed) ## first convolution: conv_1 = self.conv2d(self.block_4, self.VGG16_weights[20], self.VGG16_weights[21], "conv_1") relu_1 = tf.nn.relu(conv_1) gnorm_1 = GroupNormalization(groups=32, axis=-1, epsilon=0.1)(relu_1) drop_1 = tf.nn.dropout(gnorm_1, self.prob) ## second convolution: conv_2 = self.conv2d(drop_1, self.VGG16_weights[22], self.VGG16_weights[23], "conv_2") relu_2 = tf.nn.relu(conv_2) gnorm_2 = GroupNormalization(groups=32, axis=-1, epsilon=0.1)(relu_2) drop_2 = tf.nn.dropout(gnorm_2, self.prob) ## third convolution: conv_3 = self.conv2d(drop_2, self.VGG16_weights[24], self.VGG16_weights[25], "conv_3") relu_3 = tf.nn.relu(conv_3) gnorm_3 = GroupNormalization(groups=32, axis=-1, epsilon=0.1)(relu_3) drop_3 = tf.nn.dropout(gnorm_3, self.prob) # max pooling: pool_1 = tf.nn.max_pool(drop_3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') return pool_1
def setup_disparity_refinement_net(self): # Input image and high-res disparity map are the same size img_in = Input(shape=(self.image_shape[0], self.image_shape[1], 32)) disp_in = Input(shape=(self.image_shape[0], self.image_shape[1], 1)) # Image branch img_c1 = Conv2D(filters=16, kernel_size=(3, 3), strides=(1, 1), padding='same')(img_in) img_bn1 = GroupNormalization(groups=4)(img_c1) img_a1 = Activation(tf.nn.leaky_relu)(img_bn1) img_rn1 = resnet_block(n_filters=16, input=img_a1) img_rn2 = resnet_block(n_filters=16, dilation_rate=(2, 2), input=img_rn1) # Disparity branch disp_c1 = Conv2D(filters=16, kernel_size=(3, 3), strides=(1, 1), padding='same')(disp_in) disp_bn1 = GroupNormalization(groups=4)(disp_c1) disp_a1 = Activation(tf.nn.leaky_relu)(disp_bn1) disp_rn1 = resnet_block(n_filters=16, input=disp_a1) disp_rn2 = resnet_block(n_filters=16, dilation_rate=(2, 2), input=disp_rn1) # Glue these together along feature axis ct1 = Concatenate(axis=-1)([img_rn2, disp_rn2]) # Some more processing ref_rn1 = resnet_block(n_filters=32, dilation_rate=(4, 4), input=ct1) ref_rn2 = resnet_block(n_filters=32, dilation_rate=(8, 8), input=ref_rn1) ref_rn3 = resnet_block(n_filters=32, input=ref_rn2) ref_rn4 = resnet_block(n_filters=32, input=ref_rn3) # Reduce to single channel output (disparity residual) disp_resid_out = Conv2D(filters=1, kernel_size=(3, 3), strides=(1, 1), padding='same')(ref_rn4) self.disparity_refinement_model = Model([img_in, disp_in], disp_resid_out)
def _resnet_bottleneck_block(input, filters, k=1, strides=(1, 1)): ''' Adds a pre-activation resnet block with bottleneck layers Args: input: input tensor filters: number of output filters k: width factor strides: strides of the convolution layer Returns: a keras tensor ''' init = input channel_axis = 1 if K.image_data_format() == "channels_first" else -1 bottleneck_expand = 4 x = GroupNormalization(axis=channel_axis)(input) x = Activation('relu')(x) if strides != (1, 1) or init._keras_shape[ channel_axis] != bottleneck_expand * filters * k: init = Conv2D(bottleneck_expand * filters * k, (1, 1), padding='same', kernel_initializer='he_normal', use_bias=False, strides=strides)(x) x = Conv2D(filters * k, (1, 1), padding='same', kernel_initializer='he_normal', use_bias=False)(x) x = GroupNormalization(axis=channel_axis)(x) x = Activation('relu')(x) x = Conv2D(filters * k, (3, 3), padding='same', kernel_initializer='he_normal', use_bias=False, strides=strides)(x) x = GroupNormalization(axis=channel_axis, gamma_initializer='zeros')(x) x = Activation('relu')(x) x = Conv2D(bottleneck_expand * filters * k, (1, 1), padding='same', kernel_initializer='he_normal', use_bias=False)(x) m = add([x, init]) return m
def residual_block(input_tensor, num_of_filters, kernel_size, group_filters, residual_name): """ This function is used to create a residual block as proposed in the paper 'Vehicle Instance Segmentation From Aerial Image and Video Using a Multitask Learning Residual Fully Convolutional Network' by Mou L., Zhu X.X. (2018). Inputs: - input_tensor: Input to the residual block - num_of_filters: Number of feature maps to be generated for the residual block - kernel_size: Size of convolutional kernel to be used for the residual block - group_filters: Number of groups to be used for group normalization - residual_name: Prefix to be appended to the name of each convolutional layer in the residual block Outputs: - residual_output: Output of the residual block """ conv_1 = Conv2D(num_of_filters, (kernel_size, kernel_size), padding='same', name=residual_name + '_conv_1')(input_tensor) conv_1_gn = GroupNormalization(groups=group_filters, axis=-1, epsilon=0.1)(conv_1) conv_1_act = Activation('relu')(conv_1_gn) conv_2 = Conv2D(num_of_filters, (kernel_size, kernel_size), padding='same', name=residual_name + '_conv_2')(conv_1_act) conv_2_gn = GroupNormalization(groups=group_filters, axis=-1, epsilon=0.1)(conv_2) conv_2_act = Activation('relu')(conv_2_gn) conv_3 = Conv2D(num_of_filters, (kernel_size, kernel_size), padding='same', name=residual_name + '_conv_3')(conv_2_act) conv_3_gn = GroupNormalization(groups=group_filters, axis=-1, epsilon=0.1)(conv_3) conv_res = Conv2D(num_of_filters, (kernel_size, kernel_size), padding='same', name=residual_name + '_conv_residual')(input_tensor) conv_res_gn = GroupNormalization(groups=group_filters, axis=-1, epsilon=0.1)(conv_res) residual_output = Add()([conv_3_gn, conv_res_gn]) return residual_output
def feature_extract(inputs): x = tf.keras.layers.Conv2D(64, kernel_size=3, strides=1, padding="same", use_bias=False)(inputs) x = GroupNormalization(32)(x) x = tf.keras.layers.Activation(tf.keras.activations.relu)(x) c1 = tf.keras.layers.MaxPooling2D(pool_size=3, strides=2, padding="same")(x) c2 = _bottleneck(c1, 64, strides=1) c3 = _bottleneck(c2, 128, strides=2) c4 = _bottleneck(c3, 256, strides=2) c5 = _bottleneck(c4, 512, strides=2) p5 = tf.keras.layers.Conv2D(256, kernel_size=1, strides=1)(c5) x = tf.keras.layers.Conv2D(256, kernel_size=1, strides=1)(c4) p4 = tf.keras.layers.Add()( [tf.keras.layers.UpSampling2D(size=(2, 2))(p5), x]) p4 = tf.keras.layers.Conv2D(128, kernel_size=3, strides=1, padding="same")(p4) x = tf.keras.layers.Conv2D(128, kernel_size=1, strides=1)(c3) p3 = tf.keras.layers.Add()( [tf.keras.layers.UpSampling2D(size=(2, 2))(p4), x]) p3 = tf.keras.layers.Conv2D(128, kernel_size=3, strides=1, padding="same")(p3) return p3
def test_shared_groupnorm(): '''Test that a GN layer can be shared across different data streams. ''' # Test single layer reuse bn = GroupNormalization(input_shape=(10, ), groups=2) x1 = Input(shape=(10, )) bn(x1) x2 = Input(shape=(10, )) y2 = bn(x2) x = np.random.normal(loc=5.0, scale=10.0, size=(2, 10)) model = Model(x2, y2) assert len(model.updates) == 0 model.compile('sgd', 'mse') model.train_on_batch(x, x) # Test model-level reuse x3 = Input(shape=(10, )) y3 = model(x3) new_model = Model(x3, y3) assert len(model.updates) == 0 new_model.compile('sgd', 'mse') new_model.train_on_batch(x, x)
def getMode(): input_shape = (MAX_LEN, ) inputs = Input(shape=input_shape) emb_inputs = Embedding(NB_WORDS, EMBEDDING_DIM, weights=[embedding_matrix], trainable=True)(inputs) convLayer = Conv1D(filters=16, kernel_size=4, activation='elu', input_shape=input_shape, kernel_regularizer=regularizers.l2(1e-4), bias_regularizer=regularizers.l2(1e-4))(emb_inputs) normalizationLayer = GroupNormalization(groups=4, axis=-1)(convLayer) poolingLayer = MaxPooling1D(pool_size=4)(normalizationLayer) flattenLayer = Flatten()(poolingLayer) dropoutLayer = Dropout(0.25)(flattenLayer) denseLayer = Dense(32, activation='elu', kernel_regularizer=regularizers.l2(1e-4), bias_regularizer=regularizers.l2(1e-4))(dropoutLayer) outLayer = Dense(1, activation='sigmoid')(denseLayer) model = Model(inputs=inputs, outputs=outLayer) model.compile(loss='binary_crossentropy', optimizer=SGD(momentum=0.95, lr=0.005), metrics=[binary_accuracy]) print(model.summary()) return model
def test_groupnorm_correctness_2d_different_groups(): norm1 = GroupNormalization(axis=1, input_shape=(10, 6), groups=2) norm2 = GroupNormalization(axis=1, input_shape=(10, 6), groups=1) norm3 = GroupNormalization(axis=1, input_shape=(10, 6), groups=10) model = Sequential() model.add(norm1) model.compile(loss='mse', optimizer='rmsprop') # centered on 5.0, variance 10.0 x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10, 6)) model.fit(x, x, epochs=5, verbose=0) out = model.predict(x) out -= np.reshape(K.eval(norm1.beta), (1, 10, 1)) out /= np.reshape(K.eval(norm1.gamma), (1, 10, 1)) assert_allclose(out.mean(axis=(0, 2)), 0.0, atol=1.1e-1) assert_allclose(out.std(axis=(0, 2)), 1.0, atol=1.1e-1) model = Sequential() model.add(norm2) model.compile(loss='mse', optimizer='rmsprop') # centered on 5.0, variance 10.0 x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10, 6)) model.fit(x, x, epochs=5, verbose=0) out = model.predict(x) out -= np.reshape(K.eval(norm2.beta), (1, 10, 1)) out /= np.reshape(K.eval(norm2.gamma), (1, 10, 1)) assert_allclose(out.mean(axis=(0, 2)), 0.0, atol=1.1e-1) assert_allclose(out.std(axis=(0, 2)), 1.0, atol=1.1e-1) model = Sequential() model.add(norm3) model.compile(loss='mse', optimizer='rmsprop') # centered on 5.0, variance 10.0 x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10, 6)) model.fit(x, x, epochs=5, verbose=0) out = model.predict(x) out -= np.reshape(K.eval(norm3.beta), (1, 10, 1)) out /= np.reshape(K.eval(norm3.gamma), (1, 10, 1)) assert_allclose(out.mean(axis=(0, 2)), 0.0, atol=1.1e-1) assert_allclose(out.std(axis=(0, 2)), 1.0, atol=1.1e-1)
def block_4(self): """ Fourth block of VGG-16 with max-pooling layer removed. input: feature map of dimension 28x28x256 output: feature map of dimension 28x28x512 """ with tf.variable_scope("block_4", reuse=tf.AUTO_REUSE): tf.set_random_seed(self.seed) ## first convolution: conv_1 = self.conv2d(self.block_3, self.VGG16_weights[14], self.VGG16_weights[15], "conv_1") relu_1 = tf.nn.relu(conv_1) gnorm_1 = GroupNormalization(groups=32, axis=-1, epsilon=0.1)(relu_1) drop_1 = tf.nn.dropout(gnorm_1, self.prob) ## second convolution: conv_2 = self.conv2d(drop_1, self.VGG16_weights[16], self.VGG16_weights[17], "conv_2") relu_2 = tf.nn.relu(conv_2) gnorm_2 = GroupNormalization(groups=32, axis=-1, epsilon=0.1)(relu_2) drop_2 = tf.nn.dropout(gnorm_2, self.prob) ## third convolution: conv_3 = self.conv2d(drop_2, self.VGG16_weights[18], self.VGG16_weights[19], "conv_3") gnorm_3 = GroupNormalization(groups=32, axis=-1, epsilon=0.1)(conv_3) relu_3 = tf.nn.relu(gnorm_3) return relu_3
def block(x, channels): return tf.keras.Sequential([ tf.keras.layers.Conv2D(channels, kernel_size=3, strides=1, padding="same"), GroupNormalization(groups=channels // 4), tf.keras.layers.Activation(tf.keras.activations.relu), tf.keras.layers.MaxPooling2D((2, 1), strides=(2, 1)), ])(x)
def block_1(self): """ First block of VGG-16. input: RGB image of dimension 224x224x3 output: feature map of dimension 112x112x128 """ with tf.variable_scope("block_1", reuse=tf.AUTO_REUSE): tf.set_random_seed(self.seed) ## first convolution: conv_1 = self.conv2d(self.input_image, self.VGG16_weights[0], self.VGG16_weights[1], "conv_1") relu_1 = tf.nn.relu(conv_1) gnorm_1 = GroupNormalization(groups=32, axis=-1, epsilon=0.1)(relu_1) drop_1 = tf.nn.dropout(gnorm_1, self.prob) ## second convolution: conv_2 = self.conv2d(drop_1, self.VGG16_weights[2], self.VGG16_weights[3], "conv_2") relu_2 = tf.nn.relu(conv_2) gnorm_2 = GroupNormalization(groups=32, axis=-1, epsilon=0.1)(relu_2) drop_2 = tf.nn.dropout(gnorm_2, self.prob) # max pooling: pool_1 = tf.nn.max_pool(drop_2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') return pool_1
def _bottleneck(inputs, channels, strides): x = inputs x = tf.keras.layers.Conv2D(channels, kernel_size=1, use_bias=False)(x) x = GroupNormalization(channels // 4)(x) x = tf.keras.layers.Activation(tf.keras.activations.relu)(x) x = tf.keras.layers.Conv2D(channels, kernel_size=3, strides=strides, use_bias=False, padding="same")(x) x = GroupNormalization(channels // 4)(x) x = tf.keras.layers.Activation(tf.keras.activations.relu)(x) x = tf.keras.layers.Conv2D(2 * channels, kernel_size=1, use_bias=False)(x) x = GroupNormalization(channels // 4)(x) if strides != 1 or inputs.shape[-1] != x.shape[-1]: y = tf.keras.layers.Conv2D(2 * channels, kernel_size=1, strides=strides, use_bias=False)(inputs) res = GroupNormalization(channels // 4)(y) x = tf.keras.layers.Add()([x, res]) return tf.keras.layers.Activation(tf.keras.activations.relu)(x)
def test_groupnorm_correctness_1d(): model = Sequential() norm = GroupNormalization(input_shape=(10, ), groups=2) model.add(norm) model.compile(loss='mse', optimizer='rmsprop') # centered on 5.0, variance 10.0 x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10)) model.fit(x, x, epochs=5, verbose=0) out = model.predict(x) out -= K.eval(norm.beta) out /= K.eval(norm.gamma) assert_allclose(out.mean(), 0.0, atol=1e-1) assert_allclose(out.std(), 1.0, atol=1e-1)
def test_groupnorm_convnet(): model = Sequential() norm = GroupNormalization(axis=1, input_shape=(3, 4, 4), groups=3) model.add(norm) model.compile(loss='mse', optimizer='sgd') # centered on 5.0, variance 10.0 x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4)) model.fit(x, x, epochs=4, verbose=0) out = model.predict(x) out -= np.reshape(K.eval(norm.beta), (1, 3, 1, 1)) out /= np.reshape(K.eval(norm.gamma), (1, 3, 1, 1)) assert_allclose(np.mean(out, axis=(0, 2, 3)), 0.0, atol=1e-1) assert_allclose(np.std(out, axis=(0, 2, 3)), 1.0, atol=1e-1)
def test_that_trainable_disables_updates(): val_a = np.random.random((10, 4)) val_out = np.random.random((10, 4)) a = Input(shape=(4, )) layer = GroupNormalization(input_shape=(4, ), groups=2) b = layer(a) model = Model(a, b) model.trainable = False assert len(model.updates) == 0 model.compile('sgd', 'mse') assert len(model.updates) == 0 x1 = model.predict(val_a) model.train_on_batch(val_a, val_out) x2 = model.predict(val_a) assert_allclose(x1, x2, atol=1e-7) model.trainable = True model.compile('sgd', 'mse') assert len(model.updates) == 0 model.train_on_batch(val_a, val_out) x2 = model.predict(val_a) assert np.abs(np.sum(x1 - x2)) > 1e-5 layer.trainable = False model.compile('sgd', 'mse') assert len(model.updates) == 0 x1 = model.predict(val_a) model.train_on_batch(val_a, val_out) x2 = model.predict(val_a) assert_allclose(x1, x2, atol=1e-7)
def _construct_rgbflow_model(self): special_init_done = False for new_layer, weight_layer in zip(self.base_model.layers, self.weights_model.layers): # Find first conv layer if isinstance(new_layer, Conv2D) and not special_init_done: params = [x.copy() for x in weight_layer.get_weights()] weight = params[0] # RGB plus optical flow H, W, C, num_filters = weight.shape # Allocate new weights new_kernel_size = (H, W, self.num_channels, num_filters) new_kernels = np.zeros(new_kernel_size) # See Motion Fused Frames paper, section 3.3 for initialization technique new_kernels[:, :, :3, :] = weight new_kernels[:, :, 3:, :] = np.tile( np.mean(weight, axis=2, keepdims=True), (1, 1, 2 * self.new_length, 1)) # Allocate new weights new_kernel_size = (H, W, self.num_channels, num_filters) new_kernels = np.zeros(new_kernel_size) params[0] = new_kernels new_layer.set_weights(params) special_init_done = True else: # copy over downstream layers new_layer.set_weights(weight_layer.get_weights()) # save input layer for model conversion self.base_input_layer = self.base_model.input if self.group_norm: base_input_shape = (self.image_dim, self.image_dim, self.num_channels) gn_input = Input(shape=base_input_shape) gn_tensor = GroupNormalization(groups=self.num_channels)(gn_input) self.base_input_layer = gn_input self.base_model = self.base_model(gn_tensor) self.base_model = Model(gn_input, self.base_model) # delete canibalized weights model del self.weights_model
def test_groupnorm_convnet_no_center_no_scale(): model = Sequential() norm = GroupNormalization(axis=-1, center=False, scale=False, input_shape=(3, 4, 4), groups=2) model.add(norm) model.compile(loss='mse', optimizer='sgd') # centered on 5.0, variance 10.0 x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4)) model.fit(x, x, epochs=4, verbose=0) out = model.predict(x) assert_allclose(np.mean(out, axis=(0, 2, 3)), 0.0, atol=1e-1) assert_allclose(np.std(out, axis=(0, 2, 3)), 1.0, atol=1e-1)
def edges_and_vertices(self): """ A model that predicts all vertices of an object within an image using a similar architecture to the image encoder model. input: RGB image of dimension 224x224x3 output: downsampled vertex predictions of dimension 28x28x1 """ with tf.variable_scope("all_vertices", reuse=tf.AUTO_REUSE): ## boundary prediction: conv_1 = tf.layers.conv2d(inputs=self.representation, filters=32, padding="same", kernel_size=[3, 3], name="conv_1") gnorm_1 = GroupNormalization(groups=32, axis=-1, epsilon=0.1)(conv_1) relu_1 = tf.nn.relu(gnorm_1) flat = tf.layers.Flatten()(relu_1) E = tf.layers.dense(inputs=flat, units=784, activation=tf.nn.sigmoid) E_plus_flat = tf.concat( [flat, E], axis=1) ## combine image representation and boundary output V = tf.layers.dense(inputs=E_plus_flat, units=784, activation=tf.nn.sigmoid) ## get the index associated with the first vertex: vertex_index = tf.argmax(V, axis=1) ## one-hot encode the first vertex: vertex_encoding = tf.one_hot(indices=vertex_index, depth=784) return tf.reshape(E, [-1, 28, 28, 1]), tf.reshape( V, [-1, 28, 28, 1]), vertex_encoding
def green_block(inp, filters, data_format='channels_first', name=None): """ green_block(inp, filters, name=None) ------------------------------------ Implementation of the special residual block used in the paper. The block consists of two (GroupNorm --> ReLu --> 3x3x3 non-strided Convolution) units, with a residual connection from the input `inp` to the output. Used internally in the model. Can be used independently as well. Parameters ---------- `inp`: An keras.layers.layer instance, required The keras layer just preceding the green block. `filters`: integer, required No. of filters to use in the 3D convolutional block. The output layer of this green block will have this many no. of channels. `data_format`: string, optional The format of the input data. Must be either 'chanels_first' or 'channels_last'. Defaults to `channels_first`, as used in the paper. `name`: string, optional The name to be given to this green block. Defaults to None, in which case, keras uses generated names for the involved layers. If a string is provided, the names of individual layers are generated by attaching a relevant prefix from [GroupNorm_, Res_, Conv3D_, Relu_, ], followed by _1 or _2. Returns ------- `out`: A keras.layers.Layer instance The output of the green block. Has no. of channels equal to `filters`. The size of the rest of the dimensions remains same as in `inp`. """ inp_res = Conv3D(filters=filters, kernel_size=(1, 1, 1), strides=1, data_format=data_format, name=f'Res_{name}' if name else None)(inp) # axis=1 for channels_first data format # No. of groups = 8, as given in the paper x = GroupNormalization(groups=8, axis=1 if data_format == 'channels_first' else 0, name=f'GroupNorm_1_{name}' if name else None)(inp) x = Activation('relu', name=f'Relu_1_{name}' if name else None)(x) x = Conv3D(filters=filters, kernel_size=(3, 3, 3), strides=1, padding='same', data_format=data_format, name=f'Conv3D_1_{name}' if name else None)(x) x = GroupNormalization(groups=8, axis=1 if data_format == 'channels_first' else 0, name=f'GroupNorm_2_{name}' if name else None)(x) x = Activation('relu', name=f'Relu_2_{name}' if name else None)(x) x = Conv3D(filters=filters, kernel_size=(3, 3, 3), strides=1, padding='same', data_format=data_format, name=f'Conv3D_2_{name}' if name else None)(x) out = Add(name=f'Out_{name}' if name else None)([x, inp_res]) return out
def build_model(input_shape=(4, 160, 192, 128), output_channels=3, weight_L2=0.1, weight_KL=0.1, learning_rate=1e-4): """ build_model(input_shape=(4, 160, 192, 128), output_channels=3, weight_L2=0.1, weight_KL=0.1) ------------------------------------------- Creates the model used in the BRATS2018 winning solution by Myronenko A. (https://arxiv.org/pdf/1810.11654.pdf) Parameters ---------- `input_shape`: A 4-tuple, optional. Shape of the input image. Must be a 4D image of shape (c, H, W, D), where, each of H, W and D are divisible by 2^4, and c is divisible by 4. Defaults to the crop size used in the paper, i.e., (4, 160, 192, 128). `output_channels`: An integer, optional. The no. of channels in the output. Defaults to 3 (BraTS 2018 format). `weight_L2`: A real number, optional The weight to be given to the L2 loss term in the loss function. Adjust to get best results for your task. Defaults to 0.1. `weight_KL`: A real number, optional The weight to be given to the KL loss term in the loss function. Adjust to get best results for your task. Defaults to 0.1. Returns ------- `model`: A keras.models.Model instance The created model. """ c, H, W, D = input_shape assert len(input_shape) == 4, "Input shape must be a 4-tuple" assert (c % 4) == 0, "The no. of channels must be divisible by 4" assert (H % 16) == 0 and (W % 16) == 0 and (D % 16) == 0, \ "All the input dimensions must be divisible by 16" # ------------------------------------------------------------------------- # Encoder # ------------------------------------------------------------------------- ## Input Layer inp = Input(input_shape) ## The Initial Block x = Conv3D(filters=32, kernel_size=(3, 3, 3), strides=1, padding='same', data_format='channels_first', name='Input_x1')(inp) ## Dropout (0.2) x = Dropout(0.2)(x) ## Green Block x1 (output filters = 32) x1 = green_block(x, 32, name='x1') x = Conv3D(filters=32, kernel_size=(3, 3, 3), strides=2, padding='same', data_format='channels_first', name='Enc_DownSample_32')(x1) ## Green Block x2 (output filters = 64) x = green_block(x, 64, name='Enc_64_1') x2 = green_block(x, 64, name='x2') x = Conv3D(filters=64, kernel_size=(3, 3, 3), strides=2, padding='same', data_format='channels_first', name='Enc_DownSample_64')(x2) ## Green Blocks x2 (output filters = 128) x = green_block(x, 128, name='Enc_128_1') x3 = green_block(x, 128, name='x3') x = Conv3D(filters=128, kernel_size=(3, 3, 3), strides=2, padding='same', data_format='channels_first', name='Enc_DownSample_128')(x3) ## Green Blocks x4 (output filters = 256) x = green_block(x, 256, name='Enc_256_1') x = green_block(x, 256, name='Enc_256_2') x = green_block(x, 256, name='Enc_256_3') x4 = green_block(x, 256, name='x4') # ------------------------------------------------------------------------- # Decoder # ------------------------------------------------------------------------- ## GT (Groud Truth) Part # ------------------------------------------------------------------------- ### Green Block x1 (output filters=128) x = Conv3D(filters=128, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_GT_ReduceDepth_128')(x4) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_GT_UpSample_128')(x) x = Add(name='Input_Dec_GT_128')([x, x3]) x = green_block(x, 128, name='Dec_GT_128') ### Green Block x1 (output filters=64) x = Conv3D(filters=64, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_GT_ReduceDepth_64')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_GT_UpSample_64')(x) x = Add(name='Input_Dec_GT_64')([x, x2]) x = green_block(x, 64, name='Dec_GT_64') ### Green Block x1 (output filters=32) x = Conv3D(filters=32, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_GT_ReduceDepth_32')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_GT_UpSample_32')(x) x = Add(name='Input_Dec_GT_32')([x, x1]) x = green_block(x, 32, name='Dec_GT_32') ### Blue Block x1 (output filters=32) x = Conv3D(filters=32, kernel_size=(3, 3, 3), strides=1, padding='same', data_format='channels_first', name='Input_Dec_GT_Output')(x) x = Conv3D( filters=output_channels, # No. of tumor classes is 3 kernel_size=(1, 1, 1), strides=1, data_format='channels_first', activation='linear', name='Dec_GT_Output')(x) ### Output Block out_GT = Softmax(axis=1)(x) ## VAE (Variational Auto Encoder) Part # ------------------------------------------------------------------------- ### VD Block (Reducing dimensionality of the data) x = GroupNormalization(groups=8, axis=1, name='Dec_VAE_VD_GN')(x4) x = Activation('relu', name='Dec_VAE_VD_relu')(x) x = Conv3D(filters=16, kernel_size=(3, 3, 3), strides=2, padding='same', data_format='channels_first', name='Dec_VAE_VD_Conv3D')(x) # Not mentioned in the paper, but the author used a Flattening layer here. x = Flatten(name='Dec_VAE_VD_Flatten')(x) x = Dense(256, name='Dec_VAE_VD_Dense')(x) ### VDraw Block (Sampling) z_mean = Dense(128, name='Dec_VAE_VDraw_Mean')(x) z_var = Dense(128, name='Dec_VAE_VDraw_Var')(x) x = Lambda(sampling, name='Dec_VAE_VDraw_Sampling')([z_mean, z_var]) ### VU Block (Upsizing back to a depth of 256) # (10x12x8 neural) x = Dense((c // 4) * (H // 16) * (W // 16) * (D // 16))(x) x = Activation('relu')(x) # (1x10x12x8) x = Reshape(((c // 4), (H // 16), (W // 16), (D // 16)))(x) x = Conv3D(filters=256, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_VAE_ReduceDepth_256')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_VAE_UpSample_256')(x) ### Green Block x1 (output filters=128) x = Conv3D(filters=128, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_VAE_ReduceDepth_128')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_VAE_UpSample_128')(x) x = green_block(x, 128, name='Dec_VAE_128') ### Green Block x1 (output filters=64) x = Conv3D(filters=64, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_VAE_ReduceDepth_64')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_VAE_UpSample_64')(x) x = green_block(x, 64, name='Dec_VAE_64') ### Green Block x1 (output filters=32) x = Conv3D(filters=32, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_VAE_ReduceDepth_32')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_VAE_UpSample_32')(x) x = green_block(x, 32, name='Dec_VAE_32') ### Blue Block x1 (output filters=32) x = Conv3D(filters=32, kernel_size=(3, 3, 3), strides=1, padding='same', data_format='channels_first', name='Input_Dec_VAE_Output')(x) ### Output Block out_VAE = Conv3D(filters=4, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_VAE_Output')(x) # Build and Compile the model out = out_GT model = Model(inp, out) # Create the model model.compile(adam(lr=learning_rate), loss(input_shape, inp, out_VAE, z_mean, z_var, weight_L2=weight_L2, weight_KL=weight_KL), metrics=[dice_coefficient]) return model
def CAN_model(img_height_size, img_width_size, n_bands, num_classes, group_filters, rcm_4_filters = 512, cfm_gp_filters = 256, cfm_conv_1_filters = 512, cfm_conv_2_filters = 256, amm_3_filters = 256, l_r = 0.0001): """ This function is used to generate the Context Aggregation Network (CAN) architecture as described in the paper 'Context Aggregation Network for Semantic Labeling in Aerial Images' by Cheng W., Yang W., Wang M., Wang G., Chen J. (2019). Inputs: - img_height_size: Height of image patches to be used for model training - img_width_size: Width of image patches to be used for model training - n_bands: Number of channels contained in the image patches to be used for model training - num_classes: Number of classes to be used for semantic segmentation - group_filters: Number of groups to be used for group normalization - rcm_4_filters: Number of filters to be used for Residual Convolutional Module (RCM) on Res - 4 output - cfm_gp_filters: Number of filters to be used for global pooling component of Context Fuse Module (CFM) - cfm_conv_1_filters: Number of filters to be used for first convolutions in parallel convolutional block of CFM - cfm_conv_2_filters: Number of filters to be used for second convolutions in parallel convolutional block of CFM - amm_3_filters: Number of filters to be used for convolutional layer in Attention Mix Module (AMM) on Res - 3 output - l_r: Learning rate for the Adam optimizer Outputs: - can_model: CAN model to be trained using input parameters and network architecture """ rcm_3_filters = int(rcm_4_filters / 2) rcm_2_filters = int(rcm_3_filters / 2) rcm_1_filters = int(rcm_2_filters / 2) rcm_base_filters = int(rcm_1_filters / 2) amm_2_filters = int(amm_3_filters / 2) amm_1_filters = int(amm_2_filters / 2) resnet50_model = ResNet50(include_top = False, input_shape = (img_height_size, img_width_size, n_bands)) res_1 = resnet50_model.get_layer('activation_10').output res_2 = resnet50_model.get_layer('activation_22').output res_3 = resnet50_model.get_layer('activation_40').output res_4 = resnet50_model.get_layer('activation_49').output rcm_4_1_1_1 = Conv2D(rcm_4_filters, (1, 1), padding = 'same')(res_4) rcm_4_1_3_3 = Conv2D(rcm_4_filters, (3, 3), padding = 'same')(rcm_4_1_1_1) rcm_4_1_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_4_1_3_3) rcm_4_1_3_3_act = Activation('relu')(rcm_4_1_3_3_gn) rcm_4_1_2nd_3_3 = Conv2D(rcm_4_filters, (3, 3), padding = 'same')(rcm_4_1_3_3_act) rcm_4_1_out = Add()([rcm_4_1_1_1, rcm_4_1_2nd_3_3]) cfm_b_gp = Lambda(lambda x: K.mean(x, axis = 3)[:, :, :, None])(rcm_4_1_out) cfm_b_1_1 = Conv2D(cfm_gp_filters, (1, 1), padding = 'same')(cfm_b_gp) cfm_b_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_b_1_1) cfm_a_3_1 = Conv2D(cfm_conv_1_filters, (3, 3), padding = 'same')(rcm_4_1_out) cfm_a_3_1_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_3_1) cfm_a_3_1_act = Activation('relu')(cfm_a_3_1_gn) cfm_a_3_2 = Conv2D(cfm_conv_2_filters, (3, 3), padding = 'same')(cfm_a_3_1_act) cfm_a_3_2_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_3_2) cfm_a_7_1 = Conv2D(cfm_conv_1_filters, (7, 7), padding = 'same')(rcm_4_1_out) cfm_a_7_1_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_7_1) cfm_a_7_1_act = Activation('relu')(cfm_a_7_1_gn) cfm_a_7_2 = Conv2D(cfm_conv_2_filters, (7, 7), padding = 'same')(cfm_a_7_1_act) cfm_a_7_2_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_7_2) cfm_a_11_1 = Conv2D(cfm_conv_1_filters, (11, 11), padding = 'same')(rcm_4_1_out) cfm_a_11_1_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_11_1) cfm_a_11_1_act = Activation('relu')(cfm_a_11_1_gn) cfm_a_11_2 = Conv2D(cfm_conv_2_filters, (11, 11), padding = 'same')(cfm_a_11_1_act) cfm_a_11_2_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_11_2) cfm_a_15_1 = Conv2D(cfm_conv_1_filters, (15, 15), padding = 'same')(rcm_4_1_out) cfm_a_15_1_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_15_1) cfm_a_15_1_act = Activation('relu')(cfm_a_15_1_gn) cfm_a_15_2 = Conv2D(cfm_conv_2_filters, (15, 15), padding = 'same')(cfm_a_15_1_act) cfm_a_15_2_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(cfm_a_15_2) cfm_out = concatenate([cfm_b_gn, cfm_a_3_2_gn, cfm_a_7_2_gn, cfm_a_11_2_gn, cfm_a_15_2_gn]) rcm_4_2_1_1 = Conv2D(rcm_3_filters, (1, 1), padding = 'same')(cfm_out) rcm_4_2_3_3 = Conv2D(rcm_3_filters, (3, 3), padding = 'same')(rcm_4_2_1_1) rcm_4_2_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_4_2_3_3) rcm_4_2_3_3_act = Activation('relu')(rcm_4_2_3_3_gn) rcm_4_2_2nd_3_3 = Conv2D(rcm_3_filters, (3, 3), padding = 'same')(rcm_4_2_3_3_act) rcm_4_2_out = Add()([rcm_4_2_1_1, rcm_4_2_2nd_3_3]) rcm_3_1_1_1 = Conv2D(rcm_3_filters, (1, 1), padding = 'same')(res_3) rcm_3_1_3_3 = Conv2D(rcm_3_filters, (3, 3), padding = 'same')(rcm_3_1_1_1) rcm_3_1_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_3_1_3_3) rcm_3_1_3_3_act = Activation('relu')(rcm_3_1_3_3_gn) rcm_3_1_2nd_3_3 = Conv2D(rcm_3_filters, (3, 3), padding = 'same')(rcm_3_1_3_3_act) rcm_3_1_out = Add()([rcm_3_1_1_1, rcm_3_1_2nd_3_3]) upsam_rcm_4 = UpSampling2D(size = (2, 2), interpolation = 'bilinear')(rcm_4_2_out) amm_3_input = concatenate([rcm_3_1_out, upsam_rcm_4]) amm_3_conv = Conv2D(amm_3_filters, (3, 3), padding = 'same')(amm_3_input) amm_3_conv_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(amm_3_conv) amm_3_conv_act = Activation('relu')(amm_3_conv_gn) amm_3_gp = Lambda(lambda x: K.mean(x, axis = 3)[:, :, :, None])(amm_3_conv_act) amm_3_mult = Multiply()([amm_3_conv_act, amm_3_gp]) amm_3_out = Add()([amm_3_mult, rcm_3_1_out]) rcm_3_2_1_1 = Conv2D(rcm_2_filters, (1, 1), padding = 'same')(amm_3_out) rcm_3_2_3_3 = Conv2D(rcm_2_filters, (3, 3), padding = 'same')(rcm_3_2_1_1) rcm_3_2_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_3_2_3_3) rcm_3_2_3_3_act = Activation('relu')(rcm_3_2_3_3_gn) rcm_3_2_2nd_3_3 = Conv2D(rcm_2_filters, (3, 3), padding = 'same')(rcm_3_2_3_3_act) rcm_3_2_out = Add()([rcm_3_2_1_1, rcm_3_2_2nd_3_3]) rcm_2_1_1_1 = Conv2D(rcm_2_filters, (1, 1), padding = 'same')(res_2) rcm_2_1_3_3 = Conv2D(rcm_2_filters, (3, 3), padding = 'same')(rcm_2_1_1_1) rcm_2_1_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_2_1_3_3) rcm_2_1_3_3_act = Activation('relu')(rcm_2_1_3_3_gn) rcm_2_1_2nd_3_3 = Conv2D(rcm_2_filters, (3, 3), padding = 'same')(rcm_2_1_3_3_act) rcm_2_1_out = Add()([rcm_2_1_1_1, rcm_2_1_2nd_3_3]) upsam_rcm_3 = UpSampling2D(size = (2, 2), interpolation = 'bilinear')(rcm_3_2_out) amm_2_input = concatenate([rcm_2_1_out, upsam_rcm_3]) amm_2_conv = Conv2D(amm_2_filters, (3, 3), padding = 'same')(amm_2_input) amm_2_conv_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(amm_2_conv) amm_2_conv_act = Activation('relu')(amm_2_conv_gn) amm_2_gp = Lambda(lambda x: K.mean(x, axis = 3)[:, :, :, None])(amm_2_conv_act) amm_2_mult = Multiply()([amm_2_conv_act, amm_2_gp]) amm_2_out = Add()([amm_2_mult, rcm_2_1_out]) rcm_2_2_1_1 = Conv2D(rcm_1_filters, (1, 1), padding = 'same')(amm_2_out) rcm_2_2_3_3 = Conv2D(rcm_1_filters, (3, 3), padding = 'same')(rcm_2_2_1_1) rcm_2_2_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_2_2_3_3) rcm_2_2_3_3_act = Activation('relu')(rcm_2_2_3_3_gn) rcm_2_2_2nd_3_3 = Conv2D(rcm_1_filters, (3, 3), padding = 'same')(rcm_2_2_3_3_act) rcm_2_2_out = Add()([rcm_2_2_1_1, rcm_2_2_2nd_3_3]) rcm_1_1_1_1 = Conv2D(rcm_1_filters, (1, 1), padding = 'same')(res_1) rcm_1_1_3_3 = Conv2D(rcm_1_filters, (3, 3), padding = 'same')(rcm_1_1_1_1) rcm_1_1_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_1_1_3_3) rcm_1_1_3_3_act = Activation('relu')(rcm_1_1_3_3_gn) rcm_1_1_2nd_3_3 = Conv2D(rcm_1_filters, (3, 3), padding = 'same')(rcm_1_1_3_3_act) rcm_1_1_out = Add()([rcm_1_1_1_1, rcm_1_1_2nd_3_3]) upsam_rcm_2 = UpSampling2D(size = (2, 2), interpolation = 'bilinear')(rcm_2_2_out) amm_1_input = concatenate([rcm_1_1_out, upsam_rcm_2]) amm_1_conv = Conv2D(amm_1_filters, (3, 3), padding = 'same')(amm_1_input) amm_1_conv_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(amm_1_conv) amm_1_conv_act = Activation('relu')(amm_1_conv_gn) amm_1_gp = Lambda(lambda x: K.mean(x, axis = 3)[:, :, :, None])(amm_1_conv_act) amm_1_mult = Multiply()([amm_1_conv_act, amm_1_gp]) amm_1_out = Add()([amm_1_mult, rcm_1_1_out]) rcm_1_2_1_1 = Conv2D(rcm_base_filters, (1, 1), padding = 'same')(amm_1_out) rcm_1_2_3_3 = Conv2D(rcm_base_filters, (3, 3), padding = 'same')(rcm_1_2_1_1) rcm_1_2_3_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(rcm_1_2_3_3) rcm_1_2_3_3_act = Activation('relu')(rcm_1_2_3_3_gn) rcm_1_2_2nd_3_3 = Conv2D(rcm_base_filters, (3, 3), padding = 'same')(rcm_1_2_3_3_act) rcm_1_2_out = Add()([rcm_1_2_1_1, rcm_1_2_2nd_3_3]) deconv_2 = Conv2DTranspose(rcm_base_filters, (2, 2), strides = (2, 2), padding = 'same')(rcm_1_2_out) deconv_2_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(deconv_2) deconv_2_act = Activation('relu')(deconv_2_gn) deconv_1 = Conv2DTranspose(rcm_base_filters, (2, 2), strides = (2, 2), padding = 'same')(deconv_2_act) deconv_1_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(deconv_1) deconv_1_act = Activation('relu')(deconv_1_gn) pred_layer = Conv2D(num_classes, (1, 1), padding = 'same', activation = 'softmax')(deconv_1_act) can_model = Model(inputs = resnet50_model.input, outputs = pred_layer) can_model.compile(loss = 'categorical_crossentropy', optimizer = Adam(lr = l_r), metrics = ['categorical_crossentropy']) return can_model
def create_model(dim=(256, 256), weights=np.ones(5), split=False): f_loss = categorical_focal_loss(alpha=weights) IMG_WIDTH, IMG_HEIGHT, CHANNELS = *dim, 3 input_shape = (IMG_WIDTH, IMG_HEIGHT, CHANNELS) elu = keras.layers.ELU(alpha=1.0) # create the base pre-trained model # Load in EfficientNetB5 effnet = efn.EfficientNetB4(weights=None, include_top=False, input_shape=(IMG_WIDTH, IMG_HEIGHT, CHANNELS)) effnet.load_weights( '/media/parth/DATA/datasets/aptos_2019/efficientnet-b4_imagenet_1000_notop.h5' ) # Replace all Batch Normalization layers by Group Normalization layers for i, layer in enumerate(effnet.layers): if "batch_normalization" in layer.name: effnet.layers[i] = GroupNormalization(groups=32, axis=-1, epsilon=0.00001) if split == True: input1 = Input(input_shape) input2 = Input(input_shape) input3 = Input(input_shape) input4 = Input(input_shape) conv1 = Conv2D(16, 3, padding='same')(input1) conv2 = Conv2D(16, 3, padding='same')(input2) conv3 = Conv2D(16, 3, padding='same')(input3) conv4 = Conv2D(16, 3, padding='same')(input4) concat = concatenate([conv1, conv2, conv3, conv4]) enet_input = Conv2D(3, 3, padding='same')(concat) x = effnet(enet_input) x = GlobalAveragePooling2D()(x) x = Dense(256)(x) x = Dropout(0.25)(x) x = Dense(5)(x) predictions = Softmax()(x) model = Model(inputs=[input1, input2, input3, input4], outputs=predictions) model.compile(loss=f_loss, optimizer=RAdam(learning_rate=0.00005), metrics=[f_loss, 'acc']) print(model.summary()) return model else: x = effnet.output x = GlobalAveragePooling2D()(x) x = Dense(256)(x) x = Dropout(0.25)(x) x = Dense(5)(x) predictions = Softmax()(x) model = Model(inputs=effnet.input, outputs=predictions) model.compile(loss=f_loss, optimizer=RAdam(lr=0.00005), metrics=[f_loss, 'acc']) print(model.summary()) return model
print('Pad sequences (samples x time)') x_train = sequence.pad_sequences(x_train, maxlen=maxlen) x_test = sequence.pad_sequences(x_test, maxlen=maxlen) x_val = sequence.pad_sequences(x_val, maxlen=maxlen) print('x_train shape:', x_train.shape) print('x_test shape:', x_test.shape) print('x_val shape:', x_val.shape) # configuration matches 4.47 Million parameters with `units=600` and `64 embedding dim` print('Build model...') inputs = Input(shape=(maxlen, )) embed = Embedding(embed_size + 1, 128, input_shape=(maxlen, ))(inputs) groupNormal = GroupNormalization(groups=32, axis=-1)(embed) x_score = Convolution1D(filters=1, kernel_size=3, padding='same', activation='sigmoid')(groupNormal) x_atten = Multiply()([x_score, embed]) first_ind = IndRNN(FLAGS.units, recurrent_clip_min=-1, recurrent_clip_max=-1, dropout=0.0, recurrent_dropout=0.0, return_sequences=True)(x_atten) second_ind = IndRNN(FLAGS.units, recurrent_clip_min=-1, recurrent_clip_max=-1, dropout=0.0,
def JointNet(img_height_size = 512, img_width_size = 512, n_bands = 3, growth_rate_1 = 32, growth_rate_2 = 64, growth_rate_3 = 128, growth_rate_net_bridge = 256, group_filters = 8, l_r = 0.0001): """ This function is used to generate the JointNet architecture as described in the paper 'JointNet: A Common Neural Network for Road and Building Extraction' by Zhang Z., Wang Y. (2019). Inputs: - img_height_size: Height of image patches to be used for model training - img_width_size: Width of image patches to be used for model training - n_bands: Number of channels contained in the image patches to be used for model training - growth_rate_1: Number of filters to be used for each atrous convolution in the first dense atrous convolution block - growth_rate_2: Number of filters to be used for each atrous convolution in the second dense atrous convolution block - growth_rate_3: Number of filters to be used for each atrous convolution in the third dense atrous convolution block - growth_rate_net_bridge: Number of filters to be used for each atrous convolution in the network bridge - group_filters: Number of groups to be used for group normalization - l_r: Learning rate to be applied for the Adam optimizer Outputs: - jointnet_model: JointNet model to be trained using input parameters and network architecture """ img_input = Input(shape = (img_height_size, img_width_size, n_bands)) ec_1_ac_1 = Conv2D(growth_rate_1, (3, 3), padding = 'same', activation = 'relu')(img_input) ec_1_ac_1_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(ec_1_ac_1) ec_1_ac_1_output = concatenate([img_input, ec_1_ac_1_gn]) ec_1_ac_2 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(ec_1_ac_1_output) ec_1_ac_2_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(ec_1_ac_2) ec_1_ac_2_output = concatenate([img_input, ec_1_ac_1_gn, ec_1_ac_2_gn]) ec_1_ac_3 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(ec_1_ac_2_output) ec_1_ac_3_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(ec_1_ac_3) ec_1_ac_3_output = concatenate([img_input, ec_1_ac_1_gn, ec_1_ac_2_gn, ec_1_ac_3_gn]) ec_1_ac_4 = Conv2D(growth_rate_1, (3, 3), padding = 'same', activation = 'relu')(ec_1_ac_3_output) ec_1_ac_4_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(ec_1_ac_4) ec_1_ac_4_output = concatenate([img_input, ec_1_ac_1_gn, ec_1_ac_2_gn, ec_1_ac_3_gn, ec_1_ac_4_gn]) ec_1_ac_5 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(ec_1_ac_4_output) ec_1_ac_5_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(ec_1_ac_5) ec_1_ac_5_output = concatenate([img_input, ec_1_ac_1_gn, ec_1_ac_2_gn, ec_1_ac_3_gn, ec_1_ac_4_gn, ec_1_ac_5_gn]) ec_1_ac_6 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(ec_1_ac_5_output) ec_1_ac_6_gn = GroupNormalization(groups = group_filters, axis = -1, epsilon = 0.1)(ec_1_ac_6) ec_1_ac_6_output = concatenate([img_input, ec_1_ac_1_gn, ec_1_ac_2_gn, ec_1_ac_3_gn, ec_1_ac_4_gn, ec_1_ac_5_gn, ec_1_ac_6_gn]) ec_1_DO = Conv2D(int(4 * growth_rate_1), (1, 1), padding = 'same', activation = 'relu')(ec_1_ac_6_output) ec_1_RO = Conv2D(growth_rate_1, (1, 1), strides = (2, 2), padding = 'same', activation = 'relu')(img_input) ec_2_ac_1 = Conv2D(growth_rate_2, (3, 3), padding = 'same', activation = 'relu')(ec_1_RO) ec_2_ac_1_gn = GroupNormalization(groups = int(group_filters * 2), axis = -1, epsilon = 0.1)(ec_2_ac_1) ec_2_ac_1_output = concatenate([ec_1_RO, ec_2_ac_1_gn]) ec_2_ac_2 = Conv2D(growth_rate_2, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(ec_2_ac_1_output) ec_2_ac_2_gn = GroupNormalization(groups = int(group_filters * 2), axis = -1, epsilon = 0.1)(ec_2_ac_2) ec_2_ac_2_output = concatenate([ec_1_RO, ec_2_ac_1_gn, ec_2_ac_2_gn]) ec_2_ac_3 = Conv2D(growth_rate_2, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(ec_2_ac_2_output) ec_2_ac_3_gn = GroupNormalization(groups = int(group_filters * 2), axis = -1, epsilon = 0.1)(ec_2_ac_3) ec_2_ac_3_output = concatenate([ec_1_RO, ec_2_ac_1_gn, ec_2_ac_2_gn, ec_2_ac_3_gn]) ec_2_ac_4 = Conv2D(growth_rate_2, (3, 3), padding = 'same', activation = 'relu')(ec_2_ac_3_output) ec_2_ac_4_gn = GroupNormalization(groups = int(group_filters * 2), axis = -1, epsilon = 0.1)(ec_2_ac_4) ec_2_ac_4_output = concatenate([ec_1_RO, ec_2_ac_1_gn, ec_2_ac_2_gn, ec_2_ac_3_gn, ec_2_ac_4_gn]) ec_2_ac_5 = Conv2D(growth_rate_2, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(ec_2_ac_4_output) ec_2_ac_5_gn = GroupNormalization(groups = int(group_filters * 2), axis = -1, epsilon = 0.1)(ec_2_ac_5) ec_2_ac_5_output = concatenate([ec_1_RO, ec_2_ac_1_gn, ec_2_ac_2_gn, ec_2_ac_3_gn, ec_2_ac_4_gn, ec_2_ac_5_gn]) ec_2_ac_6 = Conv2D(growth_rate_2, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(ec_2_ac_5_output) ec_2_ac_6_gn = GroupNormalization(groups = int(group_filters * 2), axis = -1, epsilon = 0.1)(ec_2_ac_6) ec_2_ac_6_output = concatenate([ec_1_RO, ec_2_ac_1_gn, ec_2_ac_2_gn, ec_2_ac_3_gn, ec_2_ac_4_gn, ec_2_ac_5_gn, ec_2_ac_6_gn]) ec_2_DO = Conv2D(int(4 * growth_rate_2), (1, 1), padding = 'same', activation = 'relu')(ec_2_ac_6_output) ec_2_RO = Conv2D(growth_rate_2, (1, 1), strides = (2, 2), padding = 'same', activation = 'relu')(ec_1_RO) ec_3_ac_1 = Conv2D(growth_rate_3, (3, 3), padding = 'same', activation = 'relu')(ec_2_RO) ec_3_ac_1_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1, epsilon = 0.1)(ec_3_ac_1) ec_3_ac_1_output = concatenate([ec_2_RO, ec_3_ac_1_gn]) ec_3_ac_2 = Conv2D(growth_rate_3, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(ec_3_ac_1_output) ec_3_ac_2_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1, epsilon = 0.1)(ec_3_ac_2) ec_3_ac_2_output = concatenate([ec_2_RO, ec_3_ac_1_gn, ec_3_ac_2_gn]) ec_3_ac_3 = Conv2D(growth_rate_3, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(ec_3_ac_2_output) ec_3_ac_3_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1, epsilon = 0.1)(ec_3_ac_3) ec_3_ac_3_output = concatenate([ec_2_RO, ec_3_ac_1_gn, ec_3_ac_2_gn, ec_3_ac_3_gn]) ec_3_ac_4 = Conv2D(growth_rate_3, (3, 3), padding = 'same', activation = 'relu')(ec_3_ac_3_output) ec_3_ac_4_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1, epsilon = 0.1)(ec_3_ac_4) ec_3_ac_4_output = concatenate([ec_2_RO, ec_3_ac_1_gn, ec_3_ac_2_gn, ec_3_ac_3_gn, ec_3_ac_4_gn]) ec_3_ac_5 = Conv2D(growth_rate_3, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(ec_3_ac_4_output) ec_3_ac_5_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1, epsilon = 0.1)(ec_3_ac_5) ec_3_ac_5_output = concatenate([ec_2_RO, ec_3_ac_1_gn, ec_3_ac_2_gn, ec_3_ac_3_gn, ec_3_ac_4_gn, ec_3_ac_5_gn]) ec_3_ac_6 = Conv2D(growth_rate_3, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(ec_3_ac_5_output) ec_3_ac_6_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1, epsilon = 0.1)(ec_3_ac_6) ec_3_ac_6_output = concatenate([ec_2_RO, ec_3_ac_1_gn, ec_3_ac_2_gn, ec_3_ac_3_gn, ec_3_ac_4_gn, ec_3_ac_5_gn, ec_3_ac_6_gn]) ec_3_DO = Conv2D(int(4 * growth_rate_3), (1, 1), padding = 'same', activation = 'relu')(ec_3_ac_6_output) ec_3_RO = Conv2D(growth_rate_3, (1, 1), strides = (2, 2), padding = 'same', activation = 'relu')(ec_2_RO) net_bridge = Conv2D(growth_rate_net_bridge, (1, 1), padding = 'same', activation = 'relu')(ec_3_RO) net_bridge_up = UpSampling2D(size = (2, 2), interpolation = 'bilinear')(net_bridge) dc_3_input = concatenate([net_bridge_up, ec_3_DO]) dc_3_output = Conv2D(growth_rate_3, (1, 1), padding = 'same', activation = 'relu')(dc_3_input) dc_3_up = UpSampling2D(size = (2, 2), interpolation = 'bilinear')(dc_3_output) dc_2_input = concatenate([dc_3_up, ec_2_DO]) dc_2_output = Conv2D(growth_rate_2, (1, 1), padding = 'same', activation = 'relu')(dc_2_input) dc_2_up = UpSampling2D(size = (2, 2), interpolation = 'bilinear')(dc_2_output) dc_1_input = concatenate([dc_2_up, ec_1_DO]) dc_1_ac_1 = Conv2D(growth_rate_1, (3, 3), padding = 'same', activation = 'relu')(dc_1_input) dc_1_ac_1_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1, epsilon = 0.1)(dc_1_ac_1) dc_1_ac_1_output = concatenate([dc_1_input, dc_1_ac_1_gn]) dc_1_ac_2 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(dc_1_ac_1_output) dc_1_ac_2_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1, epsilon = 0.1)(dc_1_ac_2) dc_1_ac_2_output = concatenate([dc_1_input, dc_1_ac_1_gn, dc_1_ac_2_gn]) dc_1_ac_3 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(dc_1_ac_2_output) dc_1_ac_3_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1, epsilon = 0.1)(dc_1_ac_3) dc_1_ac_3_output = concatenate([dc_1_input, dc_1_ac_1_gn, dc_1_ac_2_gn, dc_1_ac_3_gn]) dc_1_ac_4 = Conv2D(growth_rate_1, (3, 3), padding = 'same', activation = 'relu')(dc_1_ac_3_output) dc_1_ac_4_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1, epsilon = 0.1)(dc_1_ac_4) dc_1_ac_4_output = concatenate([dc_1_input, dc_1_ac_1_gn, dc_1_ac_2_gn, dc_1_ac_3_gn, dc_1_ac_4_gn]) dc_1_ac_5 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (2, 2), activation = 'relu')(dc_1_ac_4_output) dc_1_ac_5_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1, epsilon = 0.1)(dc_1_ac_5) dc_1_ac_5_output = concatenate([dc_1_input, dc_1_ac_1_gn, dc_1_ac_2_gn, dc_1_ac_3_gn, dc_1_ac_4_gn, dc_1_ac_5_gn]) dc_1_ac_6 = Conv2D(growth_rate_1, (3, 3), padding = 'same', dilation_rate = (5, 5), activation = 'relu')(dc_1_ac_5_output) dc_1_ac_6_gn = GroupNormalization(groups = int(group_filters * 4), axis = -1, epsilon = 0.1)(dc_1_ac_6) dc_1_ac_6_output = concatenate([dc_1_input, dc_1_ac_1_gn, dc_1_ac_2_gn, dc_1_ac_3_gn, dc_1_ac_4_gn, dc_1_ac_5_gn, dc_1_ac_6_gn]) dc_1_DO = Conv2D(int(4 * growth_rate_1), (1, 1), padding = 'same', activation = 'relu')(dc_1_ac_6_output) class_layer = Conv2D(1, (1, 1), padding = 'same', activation = 'sigmoid')(dc_1_DO) jointnet_model = Model(inputs = img_input, outputs = class_layer) jointnet_model.compile(loss = dice_coef_loss, optimizer = Adam(lr = l_r), metrics = [dice_coef]) return jointnet_model
def global_context_block(ip, reduction_ratio=16, transform_activation='linear'): """ Adds a Global Context attention block for self attention to the input tensor. Input tensor can be or rank 3 (temporal), 4 (spatial) or 5 (spatio-temporal). # Arguments: ip: input tensor intermediate_dim: The dimension of the intermediate representation. Can be `None` or a positive integer greater than 0. If `None`, computes the intermediate dimension as half of the input channel dimension. reduction_ratio: Reduces the input filters by this factor for the bottleneck block of the transform submodule. Node: the reduction ratio must be set such that it divides the input number of channels, transform_activation: activation function to apply to the output of the transform block. Can be any string activation function availahle to Keras. # Returns: a tensor of same shape as input """ channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 ip_shape = K.int_shape(ip) # check rank and calculate the input shape if len(ip_shape) == 3: # temporal / time series data rank = 3 batchsize, dim1, channels = ip_shape elif len(ip_shape) == 4: # spatial / image data rank = 4 if channel_dim == 1: batchsize, channels, dim1, dim2 = ip_shape else: batchsize, dim1, dim2, channels = ip_shape elif len(ip_shape) == 5: # spatio-temporal / Video or Voxel data rank = 5 if channel_dim == 1: batchsize, channels, dim1, dim2, dim3 = ip_shape else: batchsize, dim1, dim2, dim3, channels = ip_shape else: raise ValueError( 'Input dimension has to be either 3 (temporal), 4 (spatial) or 5 (spatio-temporal)' ) if rank > 3: flat_spatial_dim = -1 if K.image_data_format( ) == 'channels_first' else 1 else: flat_spatial_dim = 1 """ Context Modelling Block """ # [B, ***, C] or [B, C, ***] input_flat = _spatial_flattenND(ip, rank) # [B, ..., C] or [B, C, ...] context = _convND(ip, rank, channels=1, kernel=1) # [B, ..., 1] or [B, 1, ...] context = _spatial_flattenND(context, rank) # [B, ***, 1] or [B, 1, ***] context = Softmax(axis=flat_spatial_dim)(context) # Compute context block outputs context = dot([input_flat, context], axes=flat_spatial_dim) # [B, C, 1] context = _spatial_expandND(context, rank) # [B, C, 1...] or [B, 1..., C] """ Transform block """ # Transform bottleneck # [B, C // R, 1...] or [B, 1..., C // R] transform = _convND(context, rank, channels // reduction_ratio, kernel=1) # Group normalization acts as Layer Normalization when groups = 1 transform = GroupNormalization(groups=1, axis=channel_dim)(transform) transform = Activation('relu')(transform) # Transform output block # [B, C, 1...] or [B, 1..., C] transform = _convND(transform, rank, channels, kernel=1) transform = Activation(transform_activation)(transform) # apply context transform out = add([ip, transform]) return out
def build_model(input_shape=(4, 160, 192, 128)): """ build_model(input_shape=(4, 160, 192, 128)) ------------------------------------------- Creates the model used in the BRATS2018 winning solution by Myronenko A. (https://arxiv.org/pdf/1810.11654.pdf) Parameters ---------- `input_shape`: A 4-tuple, optional. Shape of the input image. Must be a 4D image of shape (c, H, W, D), where, each of H, W and D are divisible by 2^4. Defaults to the crop size used in the paper, i.e., (4, 160, 192, 128). Returns ------- `model`: A keras.models.Model instance The created model. """ c, H, W, D = input_shape assert len(input_shape) == 4, "Input shape must be a 4-tuple" assert ~(H % 16) and ~(W % 16) and ~(D % 16), \ "All the input dimensions must be divisible by 16" # ------------------------------------------------------------------------- # Encoder # ------------------------------------------------------------------------- ## Input Layer inp = Input(input_shape) ## The Initial Block x = Conv3D(filters=32, kernel_size=(3, 3, 3), strides=1, padding='same', data_format='channels_first', name='Input_x1')(inp) ## Dropout (0.2) x = Dropout(0.2)(x) ## Green Block x1 (output filters = 32) x1 = green_block(x, 32, name='x1') x = Conv3D(filters=32, kernel_size=(3, 3, 3), strides=2, padding='same', data_format='channels_first', name='Enc_DownSample_32')(x1) ## Green Block x2 (output filters = 64) x = green_block(x, 64, name='Enc_64_1') x2 = green_block(x, 64, name='x2') x = Conv3D(filters=64, kernel_size=(3, 3, 3), strides=2, padding='same', data_format='channels_first', name='Enc_DownSample_64')(x2) ## Green Blocks x2 (output filters = 128) x = green_block(x, 128, name='Enc_128_1') x3 = green_block(x, 128, name='x3') x = Conv3D(filters=128, kernel_size=(3, 3, 3), strides=2, padding='same', data_format='channels_first', name='Enc_DownSample_128')(x3) ## Green Blocks x4 (output filters = 256) x = green_block(x, 256, name='Enc_256_1') x = green_block(x, 256, name='Enc_256_2') x = green_block(x, 256, name='Enc_256_3') x4 = green_block(x, 256, name='x4') # ------------------------------------------------------------------------- # Decoder # ------------------------------------------------------------------------- ## GT (Groud Truth) Part # ------------------------------------------------------------------------- ### Green Block x1 (output filters=128) x = Conv3D(filters=128, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_GT_ReduceDepth_128')(x4) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_GT_UpSample_128')(x) x = Add(name='Input_Dec_GT_128')([x, x3]) x = green_block(x, 128, name='Dec_GT_128') ### Green Block x1 (output filters=64) x = Conv3D(filters=64, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_GT_ReduceDepth_64')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_GT_UpSample_64')(x) x = Add(name='Input_Dec_GT_64')([x, x2]) x = green_block(x, 64, name='Dec_GT_64') ### Green Block x1 (output filters=32) x = Conv3D(filters=32, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_GT_ReduceDepth_32')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_GT_UpSample_32')(x) x = Add(name='Input_Dec_GT_32')([x, x1]) x = green_block(x, 32, name='Dec_GT_32') ### Blue Block x1 (output filters=32) x = Conv3D(filters=32, kernel_size=(3, 3, 3), strides=1, padding='same', data_format='channels_first', name='Input_Dec_GT_Output')(x) ### Output Block out_GT = Conv3D( filters=3, # No. of tumor classes is 3 kernel_size=(1, 1, 1), strides=1, data_format='channels_first', activation='sigmoid', name='Dec_GT_Output')(x) ## VAE (Variational Auto Encoder) Part # ------------------------------------------------------------------------- ### VD Block (Reducing dimensionality of the data) x = GroupNormalization(groups=8, axis=1, name='Dec_VAE_VD_GN')(x4) x = Activation('relu', name='Dec_VAE_VD_relu')(x) x = Conv3D(filters=16, kernel_size=(3, 3, 3), strides=2, padding='same', data_format='channels_first', name='Dec_VAE_VD_Conv3D')(x) # Not mentioned in the paper, but the author used a Flattening layer here. x = Flatten(name='Dec_VAE_VD_Flatten')(x) x = Dense(256, name='Dec_VAE_VD_Dense')(x) ### VDraw Block (Sampling) z_mean = Dense(128, name='Dec_VAE_VDraw_Mean')(x) z_var = Dense(128, name='Dec_VAE_VDraw_Var')(x) x = Lambda(sampling, name='Dec_VAE_VDraw_Sampling')([z_mean, z_var]) ### VU Block (Upsizing back to a depth of 256) x = Dense(1 * 10 * 12 * 8)(x) x = Activation('relu')(x) x = Reshape((1, 10, 12, 8))(x) x = Conv3D(filters=256, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_VAE_ReduceDepth_256')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_VAE_UpSample_256')(x) ### Green Block x1 (output filters=128) x = Conv3D(filters=128, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_VAE_ReduceDepth_128')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_VAE_UpSample_128')(x) x = green_block(x, 128, name='Dec_VAE_128') ### Green Block x1 (output filters=64) x = Conv3D(filters=64, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_VAE_ReduceDepth_64')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_VAE_UpSample_64')(x) x = green_block(x, 64, name='Dec_VAE_64') ### Green Block x1 (output filters=32) x = Conv3D(filters=32, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_VAE_ReduceDepth_32')(x) x = UpSampling3D(size=2, data_format='channels_first', name='Dec_VAE_UpSample_32')(x) x = green_block(x, 32, name='Dec_VAE_32') ### Blue Block x1 (output filters=32) x = Conv3D(filters=32, kernel_size=(3, 3, 3), strides=1, padding='same', data_format='channels_first', name='Input_Dec_VAE_Output')(x) ### Output Block out_VAE = Conv3D(filters=4, kernel_size=(1, 1, 1), strides=1, data_format='channels_first', name='Dec_VAE_Output')(x) # Build and Compile the model out = out_GT model = Model(inp, out) # Create the model model.compile(adam(lr=1e-4), loss(input_shape, inp, out_VAE, z_mean, z_var), metrics=['accuracy']) return model