def conv_block_3d(m, num_kernels, kernel_size, strides, padding, activation,
dropout, data_format, bn):
"""
Bulding block with convolutional layers for one level.
:param m: model
:param num_kernels: number of convolution filters on the particular level, positive integer
:param kernel_size: size of the convolution kernel, tuple of two positive integers
:param strides: strides values, tuple of two positive integers
:param padding: used padding by convolution, takes values: 'same' or 'valid'
:param activation: activation_function after every convolution
:param dropout: percentage of weights to be dropped, float between 0 and 1
:param data_format: ordering of the dimensions in the inputs, takes values: 'channel_first' or 'channel_last'
:param bn: weather to use Batch Normalization layers after each convolution layer, True for use Batch Normalization,
False do not use Batch Normalization
:return: model
"""
n = Convolution3D(num_kernels,
kernel_size,
strides=strides,
activation=activation,
padding=padding,
data_format=data_format)(m)
n = BatchNormalization()(n) if bn else n
n = Dropout(dropout)(n)
n = Convolution3D(num_kernels,
kernel_size,
strides=strides,
activation=activation,
padding=padding,
data_format=data_format)(n)
n = BatchNormalization()(n) if bn else n
return n
def bottleneck_layer(self, input_tensor, growth_k):
out_bn_1 = InstanceNormalization(axis=self.concat_axis,
center=True)(input_tensor)
out_relu_1 = Activation('relu')(out_bn_1)
out_conv_1 = Convolution3D(filters=growth_k * 4,
kernel_size=(1, 1, 1),
strides=(1, 1, 1),
padding='same',
use_bias=True,
kernel_initializer='he_normal',
kernel_regularizer=None)(out_relu_1)
out_bn_2 = InstanceNormalization(axis=self.concat_axis,
center=True)(out_conv_1)
out_relu_2 = Activation('relu')(out_bn_2)
output_tensor = Convolution3D(filters=growth_k,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
use_bias=True,
kernel_initializer='he_normal',
kernel_regularizer=None)(out_relu_2)
return output_tensor
def convolutional_block(input_layer, n_filters, activation=None, filter_size=(3, 3, 3), strides=(1, 1, 1),
padding='same', data_format='channels_last', batch_normalization=False, dropout=None,
l1=None, l2=None, axis=4, use_bias=True, bn_name=None, conv_name=None):
"""
Block of one convolutional layer with possible activation and batch normalization.
:param input_layer: Input layer to the convolution. Keras layer.
:param n_filters: Number of filters in the particular convolutional layer. Positive integer.
:param activation: Activation_function after every convolution. String activation name.
:param filter_size: Size of the convolution filter (kernel). Tuple of 3 positive integers.
:param strides: Strides values. Tuple of 3 positive integers.
:param padding: Used padding by convolution. Takes values: 'same' or 'valid'.
:param data_format: Ordering of the dimensions in the inputs. Takes values: 'channel_first' or 'channel_last'.
:param batch_normalization: If set to True, will use Batch Normalization layers after each convolution layer.
:param dropout: percentage of weights to be dropped, float between 0 and 1.
:param l1: L1 regularization.
:param l2: L2 regularization.
:param axis: Axis for batch normalization.
:param use_bias:
:param bn_name: Name of the batch normalization layer. String.
:param conv_name:
:return: Keras layer.
"""
if l1 is not None:
if l2 is not None:
layer = Convolution3D(filters=n_filters, kernel_size=filter_size, strides=strides, padding=padding,
data_format=data_format, kernel_regularizer=regularizers.l1_l2(l1=l1, l2=l2),
use_bias=use_bias, name=conv_name)(input_layer)
else:
layer = Convolution3D(filters=n_filters, kernel_size=filter_size, strides=strides, padding=padding,
data_format=data_format, kernel_regularizer=regularizers.l1(l1),
use_bias=use_bias, name=conv_name)(input_layer)
else:
if l2 is not None:
layer = Convolution3D(filters=n_filters, kernel_size=filter_size, strides=strides, padding=padding,
data_format=data_format, kernel_regularizer=regularizers.l2(l2), use_bias=use_bias,
name=conv_name)(
input_layer)
else:
layer = Convolution3D(filters=n_filters, kernel_size=filter_size, strides=strides, padding=padding,
data_format=data_format, use_bias=use_bias, name=conv_name)(input_layer)
if batch_normalization:
layer = BatchNormalization(axis=axis, name=bn_name)(
layer) # the axis that should be normalized (typically the features axis), integer.
# Layer input shape: (samples, conv_dim1, conv_dim2, conv_dim3, channels)` if data_format='channels_last'
if activation is not None:
layer = Activation(activation=activation)(layer)
if dropout is not None:
layer = Dropout(dropout)(layer)
return layer
def get_unet_3d(input_dim,
num_channels,
dropout,
activation='relu',
final_activation='sigmoid',
kernel_size=(3, 3, 3),
pool_size=(2, 2, 2),
strides=(1, 1, 1),
num_kernels=None,
concat_axis=-1,
data_format='channels_last',
padding='same',
bn=True):
# build model
# specify the input shape
inputs = Input((input_dim[0], input_dim[1], input_dim[2], num_channels))
# BN for inputs
layer = BatchNormalization()(inputs)
# --- Down-scale side
conv_down, residuals = down_scale_path_3d(layer, num_kernels, kernel_size,
strides, pool_size, padding,
activation, dropout, data_format,
bn)
# Fully connected 1
fl1 = Convolution3D(num_kernels[-1], (1, 1, 1),
strides=(1, 1, 1),
padding="same",
activation="relu",
data_format=data_format)(conv_down)
# Fully connected 2
fl2 = Convolution3D(num_kernels[-1], (1, 1, 1),
strides=(1, 1, 1),
padding="same",
activation="relu",
data_format=data_format)(fl1)
# --- Up-scale side
outputs = up_scale_path_3d(fl2, residuals, num_kernels, kernel_size,
strides, pool_size, concat_axis, padding,
activation, final_activation, dropout,
data_format, bn)
model = Model(inputs=inputs, outputs=outputs)
# print out model summary to console
model.summary()
return model
def up_scale_path_3d(inputs, residuals, num_kernels, kernel_size, strides,
pool_size, concat_axis, padding, activation,
final_activation, dropout, data_format, bn):
# UP-SAMPLING PART (right side of the U-net)
# layers on each level: upsampling2d -> concatenation with feature maps of corresponding level from down-sampling
# part -> convolution2d -> dropout -> convolution2d
# final convolutional layer maps feature maps to desired number of classes
conv_up = inputs
output_dim = residuals["conv_0"].shape
for i in range(len(num_kernels) - 1):
# level i
conv_up = up_concat_block_3d(
conv_up, residuals["conv_" + str(len(num_kernels) - i - 2)],
pool_size, concat_axis, data_format)
conv_up = conv_block_3d(conv_up, num_kernels[len(num_kernels) - i - 2],
kernel_size, strides, padding, activation,
dropout, data_format, bn)
final_conv = Convolution3D(1,
1,
strides=strides,
activation=final_activation,
padding=padding,
data_format=data_format)(conv_up)
return final_conv
def testOutput(self):
with self.assertRaises(ValueError) as cm:
self.classifier.add(Convolution1D(64,0,padding="same",input_shape=(32,32,1),activation='relu'))
print(cm.expected)
with self.assertRaises(ValueError) as cm:
self.classifier.add(Convolution2D(64,0,padding="same",input_shape=(32,32,1),activation='relu'))
print(cm.expected)
with self.assertRaises(ValueError) as cm:
self.classifier.add(Convolution3D(64,0,padding="same",input_shape=(32,32,1),activation='relu'))
print(cm.expected)
with self.assertRaises(ValueError) as cm:
self.classifier.add(Conv2DTranspose(64,0,padding="same",input_shape=(32,32,1),activation='relu'))
print(cm.expected)
with self.assertRaises(ValueError) as cm:
self.classifier.add(Conv3DTranspose(64,0,padding="same",input_shape=(32,32,1),activation='relu'))
print(cm.expected)
with self.assertRaises(ValueError) as cm:
self.classifier.add(SeparableConv1D(64,0,padding="same",input_shape=(32,32,1),activation='relu'))
print(cm.expected)
with self.assertRaises(ValueError) as cm:
self.classifier.add(SeparableConv2D(64,0,padding="same",input_shape=(32,32,1),activation='relu'))
print(cm.expected)
with self.assertRaises(ValueError) as cm:
self.classifier.add(DepthwiseConv2D(0,padding="same",input_shape=(32,32,1),activation='relu'))
print(cm.expected)
def init_model(model_type: str) -> Sequential:
# input size definition
image_rows, image_columns = 64, 64
if model_type == "CASME":
image_depth = CASME_DEPTH
else:
image_depth = SMIC_DEPTH
# definition of models
model = Sequential()
model.add(Convolution3D(32, (3, 3, 15), input_shape=(image_rows, image_columns, image_depth, 1), activation='relu'))
model.add(MaxPooling3D(pool_size=(3, 3, 3)))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(128, kernel_initializer='normal', activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(3, kernel_initializer='normal'))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='SGD', metrics=['accuracy'])
model.summary()
print("loading weights...")
if model_type == "CASME":
model.load_weights(WEIGHTS1)
else:
model.load_weights(WEIGHTS2)
return model
def up_scale_path_ds_3d(inputs, residuals, num_kernels, kernel_size, strides,
pool_size, concat_axis, padding, activation,
final_activation, dropout, data_format, bn):
# UP-SAMPLING PART (right side of the U-net)
# layers on each level: upsampling2d -> concatenation with feature maps of corresponding level from down-sampling
# part -> convolution2d -> dropout -> convolution2d
# final convolutional layer maps feature maps to desired number of classes
outputs = []
conv_up = inputs
output_dim = residuals["conv_0"].shape
for i in range(len(num_kernels) - 1):
# level i
conv_up = up_concat_block_3d(
conv_up, residuals["conv_" + str(len(num_kernels) - i - 2)],
pool_size, concat_axis, data_format)
conv_up = conv_block_3d(conv_up, num_kernels[len(num_kernels) - i - 2],
kernel_size, strides, padding, activation,
dropout, data_format, bn)
if i < len(num_kernels) - 2:
# get level prediction
output = UpSampling3D(
size=(int(output_dim[1].value / conv_up.shape[1].value),
int(output_dim[2].value / conv_up.shape[2].value),
int(output_dim[3].value / conv_up.shape[3].value)),
data_format=data_format)(conv_up)
output = Convolution3D(1,
1,
strides=strides,
activation=final_activation,
padding=padding,
data_format=data_format)(output)
outputs.append(output)
final_conv = Convolution3D(1,
1,
strides=strides,
activation=final_activation,
padding=padding,
data_format=data_format)(conv_up)
outputs.append(final_conv)
return outputs
def build(classes=1):
model = Sequential()
# Conv layer 1
model.add(
Convolution3D(
# input_shape = (33, 33, 33, 1),
filters=128,
kernel_size=5,
padding='valid', # Padding method
data_format='channels_first'))
model.add(LeakyReLU(alpha=0.1))
# Dropout 1
model.add(Dropout(0.2))
# Conv layer 2
model.add(
Convolution3D(
filters=128,
kernel_size=3,
padding='valid', # Padding method
data_format='channels_first'))
model.add(LeakyReLU(alpha=0.1))
# Maxpooling 1
model.add(
MaxPooling3D(
pool_size=(2, 2, 2),
strides=None,
padding='valid', # Padding method
data_format='channels_first'))
# Dropout 2
model.add(Dropout(0.4))
# FC 1
model.add(Flatten())
model.add(Dense(128)) # TODO changed to 64 for the CAM
model.add(LeakyReLU(alpha=0.1))
# Dropout 3
model.add(Dropout(0.4))
# Fully connected layer 2 to shape (1) for 2 classes
model.add(Dense(classes))
if classes == 1:
model.add(Activation('sigmoid'))
else:
model.add(Activation('softmax'))
return model
def transition_layer(self, input_tensor, theta=0.5):
nb_channel = int(input_tensor.shape[-1])
out_bn_1 = InstanceNormalization(axis=self.concat_axis,
center=True)(input_tensor)
out_conv_1 = Convolution3D(filters=int(nb_channel * theta),
kernel_size=(1, 1, 1),
strides=(1, 1, 1),
padding='same',
use_bias=True,
kernel_initializer='he_normal',
kernel_regularizer=None)(out_bn_1)
return out_conv_1
def last_label_layer(self, input_tensor):
bn_1 = InstanceNormalization(axis=self.concat_axis,
center=True)(input_tensor)
relu_1 = Activation('relu')(bn_1)
conv_1 = Convolution3D(filters=256,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
use_bias=True,
kernel_initializer='he_normal',
kernel_regularizer=None)(relu_1)
bn_2 = InstanceNormalization(axis=self.concat_axis,
center=True)(conv_1)
relu_2 = Activation('relu')(bn_2)
conv_2 = Convolution3D(filters=512,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
use_bias=True,
kernel_initializer='he_normal',
kernel_regularizer=None)(relu_2)
return subpixel_8_3D_wChannel(out_c=1)(conv_2)
def CNN(neighbor):
model = Sequential()
model.add(
Convolution3D(32, (2, 2, 2),
use_bias=True,
padding='SAME',
strides=1,
activation='relu',
input_shape=(34, neighbor, neighbor, 7)))
model.add(
Convolution3D(64, (2, 2, 2),
use_bias=True,
padding='SAME',
strides=1,
activation='relu'))
model.add(MaxPooling3D(pool_size=(3, 3, 3)))
model.add(Flatten())
model.add(Dense(256, activation='relu'))
model.add(Dense(256, activation='relu'))
model.add(Dense(1, activation='relu'))
# optimize
adam = Adam()
model.compile(loss='mse', optimizer=adam)
return model
def ACNN(x, nfeat, nfeat_fac, depth, fc, dropout_rate, l2_weight, **kwarg):
for i in range(depth):
x = Convolution3D(
nfeat, (3, 3, 3),
kernel_regularizer=tf.keras.regularizers.l2(l2_weight))(x)
x = ReLU()(x)
x = MaxPooling3D(pool_size=(2, 2, 2))(x)
nfeat *= nfeat_fac
x = Flatten()(x)
for n_unit in fc:
x = Dense(n_unit,
activation='relu',
kernel_regularizer=tf.keras.regularizers.l2(l2_weight))(x)
x = Dropout(dropout_rate)(x)
return x
def CNN(neighbor):
model = Sequential()
model.add(Convolution3D(32, (2,2,2), use_bias=True, padding='SAME', strides=1, activation='relu', input_shape=(34, neighbor, neighbor, 7)))
model.add(Convolution3D(64, (2,2,2), use_bias=True, padding='SAME', strides=1, activation='relu'))
model.add(Convolution3D(128, (2,2,2), use_bias=True, padding='SAME', strides=1, activation='relu'))
model.add(MaxPooling3D(pool_size=(2,2,2)))
model.add(Convolution3D(32, (2,2,2), use_bias=True, padding='SAME', strides=1, activation='relu'))
model.add(Convolution3D(64, (2,2,2), use_bias=True, padding='SAME', strides=1, activation='relu'))
model.add(Convolution3D(128, (2,2,2), use_bias=True, padding='SAME', strides=1, activation='relu'))
model.add(MaxPooling3D(pool_size=(2,2,2)))
model.add(Flatten())
model.add(Dense(1, activation='relu'))
model.compile(optimizer='adam', loss='mean_squared_error')
def DenseNet_v3(self, input):
# growth_k = 8
growth_k = 4
theta = 0.5
init_conv_0 = Convolution3D(filters=growth_k * 2,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
use_bias=True,
kernel_initializer='he_normal',
kernel_regularizer=None)(
input) # 200 x 200 x 200
init_bn_0 = InstanceNormalization(axis=self.concat_axis,
center=False)(init_conv_0)
init_relu_0 = Activation('relu')(init_bn_0)
init_conv_1 = Convolution3D(filters=growth_k * 2,
kernel_size=(7, 7, 7),
strides=(2, 2, 2),
padding='same',
use_bias=True,
kernel_initializer='he_normal',
kernel_regularizer=None)(
init_relu_0) # 100 x 100 x 100
nb_layers = 12
denseblk_1 = self.dense_block(init_conv_1,
nb_layers=nb_layers,
growth_k=growth_k)
transit_1 = self.transition_layer(denseblk_1, theta=theta)
bsize, zz, yy, xx, c = transit_1.get_shape().as_list()
transit_1 = Convolution3D(filters=c,
kernel_size=(3, 3, 3),
strides=(2, 2, 2),
padding='same',
use_bias=True,
kernel_initializer='he_normal',
kernel_regularizer=None)(
transit_1) # 50 x 50 x50
nb_layers = 24
denseblk_2 = self.dense_block(transit_1,
nb_layers=nb_layers,
growth_k=growth_k)
transit_2 = self.transition_layer(denseblk_2, theta=theta)
bsize, zz, yy, xx, c = transit_2.get_shape().as_list()
transit_2 = Convolution3D(filters=c,
kernel_size=(3, 3, 3),
strides=(2, 2, 2),
padding='same',
use_bias=True,
kernel_initializer='he_normal',
kernel_regularizer=None)(
transit_2) # 25 x 25 x 25
nb_layers = 18
denseblk_3 = self.dense_block(transit_2,
nb_layers=nb_layers,
growth_k=growth_k)
label_0 = self.last_label_layer(denseblk_3)
label_1 = self.last_label_layer(denseblk_3)
label_2 = self.last_label_layer(denseblk_3)
concat_1 = Concatenate()([label_0, label_1, label_2, init_relu_0])
last_conv_0 = Convolution3D(filters=growth_k,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
use_bias=True,
kernel_initializer='he_normal',
kernel_regularizer=None)(
concat_1) # 200 x 200 x 200
last_bn_0 = InstanceNormalization(axis=self.concat_axis,
center=False)(last_conv_0)
last_relu_0 = Activation('relu')(last_bn_0)
last_conv_1 = Convolution3D(filters=3,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
use_bias=True,
kernel_initializer='he_normal',
kernel_regularizer=None)(last_relu_0)
return last_conv_1
def compose_vgg_model(anchors, base, cut_layers, input_shape, out_channels,
use_dropout):
anchor_layers = []
input_layer = Input(shape=input_shape, name='input_3d')
x = input_layer
weights_to_copy = {}
for layer in base.layers[:len(base.layers) - cut_layers]:
if layer.__class__ == Conv2D:
ws = layer.get_weights()
ws[0] = np.sum(ws[0], axis=2,
keepdims=True) if ws[0].shape[2] == 3 else ws[0]
ws[0] = np.expand_dims(ws[0], 2)
shp = ws[0].shape
nm = layer.name + '_3d'
print(f'Creating Conv3D layer {nm} with weights shape {shp}')
ks = shp[:2] + (1, )
new_layer = Convolution3D(filters=shp[-1],
kernel_size=ks,
padding='same',
activation='relu',
name=nm)
weights_to_copy[nm] = ws
x = new_layer(x)
if layer.name in anchors:
anchor_layers.append(x)
elif layer.__class__ == MaxPooling2D:
nm = layer.name + '_3d'
print('Creating MaxPooling3D layer ' + nm)
x = MaxPooling3D((2, 2, 1), name=nm)(x)
anchor_layers = anchor_layers[:-1]
for i, _ in enumerate(anchor_layers):
x = UpSampling3D(size=(2, 2, 1), name=f'up_{i}')(x)
layer = anchor_layers[-1 - i]
x = Concatenate(axis=-1)([layer, x])
filters = layer.shape[-1]
nm = f'conv_up{i}-1'
x = Convolution3D(filters=filters,
kernel_size=(3, 3, 3),
padding='same',
activation='relu',
name=nm)(x)
nm = f'conv_up{i}-2'
x = Convolution3D(filters=filters,
kernel_size=(3, 3, 1),
padding='same',
activation='relu',
name=nm)(x)
if use_dropout:
x = Dropout(0.5)(x)
output = Convolution3D(filters=out_channels,
kernel_size=1,
padding='same',
activation='sigmoid',
name='predictions')(x)
model = Model(input_layer, output)
for layer in model.layers:
if layer.name in weights_to_copy:
print('Copying weights to ' + layer.name)
layer.set_weights(weights_to_copy[layer.name])
layer.trainable = False
return model
def get_unet_us_untouched(image_shape):
with tf.device("/gpu:0"):
inputs = Input(
(image_shape[0], image_shape[1], image_shape[2], 1)) # tensor-flow
conv1 = Convolution3D(32, (3, 3, 3), activation='relu',
padding='same')(inputs)
conv1 = Convolution3D(32, (3, 3, 3), activation='relu',
padding='same')(conv1)
pool1 = MaxPooling3D(pool_size=(2, 2, 2))(conv1)
conv2 = Convolution3D(64, (3, 3, 3), activation='relu',
padding='same')(pool1)
conv2 = Convolution3D(64, (3, 3, 3), activation='relu',
padding='same')(conv2)
pool2 = MaxPooling3D(pool_size=(2, 2, 2))(conv2)
conv3 = Convolution3D(128, (3, 3, 3),
activation='relu',
padding='same')(pool2)
conv3 = Convolution3D(128, (3, 3, 3),
activation='relu',
padding='same')(conv3)
pool3 = MaxPooling3D(pool_size=(2, 2, 2))(conv3)
conv4 = Convolution3D(256, (3, 3, 3),
activation='relu',
padding='same')(pool3)
conv4 = Convolution3D(256, (3, 3, 3),
activation='relu',
padding='same')(conv4)
pool4 = MaxPooling3D(pool_size=(2, 2, 2))(conv4)
# with tf.device ("/gpu:1"):
conv5 = Convolution3D(512, (3, 3, 3),
activation='relu',
padding='same')(pool4)
conv5 = Convolution3D(512, (3, 3, 3),
activation='relu',
padding='same')(conv5)
up6 = concatenate([
Conv3DTranspose(256, (2, 2, 2), strides=(2, 2, 2),
padding='same')(conv5), conv4
],
axis=4)
conv6 = Convolution3D(256, (3, 3, 3),
activation='relu',
padding='same')(up6)
conv6 = Convolution3D(256, (3, 3, 3),
activation='relu',
padding='same')(conv6)
up7 = concatenate([
Conv3DTranspose(128, (2, 2, 2), strides=(2, 2, 2),
padding='same')(conv6), conv3
],
axis=4)
conv7 = Convolution3D(128, (3, 3, 3),
activation='relu',
padding='same')(up7)
conv7 = Convolution3D(128, (3, 3, 3),
activation='relu',
padding='same')(conv7)
up8 = concatenate([
Conv3DTranspose(64, (2, 2, 2), strides=(2, 2, 2),
padding='same')(conv7), conv2
],
axis=4)
conv8 = Convolution3D(64, (3, 3, 3), activation='relu',
padding='same')(up8)
conv8 = Convolution3D(64, (3, 3, 3), activation='relu',
padding='same')(conv8)
up9 = concatenate([
Conv3DTranspose(32, (2, 2, 2), strides=(2, 2, 2),
padding='same')(conv8), conv1
],
axis=4)
conv9 = Convolution3D(32, (3, 3, 3), activation='relu',
padding='same')(up9)
conv9 = Convolution3D(32, (3, 3, 3), activation='relu',
padding='same')(conv9)
conv10 = Convolution3D(1, (1, 1, 1), activation='sigmoid')(conv9)
model = Model(inputs=[inputs], outputs=[conv10])
# model.compile(optimizer=Adam(lr=1e-5), loss=dice_coef_loss, metrics=[dice_coef])
return model
X = data_tem['X']
Y = data_tem['Y']
index = np.arange(X.shape[0])
np.random.shuffle(index)
X = X[index, :, :, :]
Y = Y[index, :]
X = X.reshape(X.shape[0], 10, 36, 36, 1)
print(X.shape, Y.shape)
#X=X[:1000,:,:,:,:];Y=Y[:1000,:];print(X.shape,Y.shape)
#builf CNN
model = keras.Sequential()
#Conv layer1 output shape (32,10,36,36)
model.add(
Convolution3D(batch_input_shape=(None, 10, 36, 36, 1),
filters=32,
kernel_size=(6, 6, 6),
strides=2,
padding='same',
data_format="channels_last"))
model.add(BatchNormalization())
model.add(Activation('relu'))
#Pooling layer 1(max pooling) output shape(32,3,6,6)
model.add(
MaxPooling3D(pool_size=(2, 6, 6),
strides=2,
padding='same',
data_format='channels_last'))
model.add(
Convolution3D(filters=16,
kernel_size=(6, 6, 6),
strides=2,
padding='same',
def get_bravenet(input_shapes,
n_classes=1,
activation='relu',
final_activation='softmax',
optimizer=None,
learning_rate=1e-4,
n_base_filters=32,
depth=3,
dropout=None,
l1=None,
l2=None,
loss_function=None,
metrics='accuracy',
filter_size=(3, 3, 3),
pool_size=(2, 2, 2),
concat_axis=-1,
data_format='channels_last',
padding='same',
batch_normalization=False,
deconvolution=False):
"""
Defines the architecture of the brave-net.
:param input_shapes: List of shapes of the input data. List of tuples of 4 positive integers (x_size, y_size,
z_size, n_channels).
The x, y, and z sizes must be divisible by the pool size to the power of the depth of the net, that is
pool_size^depth.
:param n_classes: Number of class labels that the model is learning. Positive integer.
:param activation: Activation_function after every convolution. String activation name.
:param final_activation: Activation_function of the final layer. String activation name.
:param optimizer: Optimization algorithm for updating the weights and bias values.
:param learning_rate: Learning_rate of the optimizer. Float.
:param dropout: Percentage of weights to be dropped. Float between 0 and 1.
:param l1: L1 regularization.
:param l2: L2 regularization.
:param n_base_filters: The number of filters that the first layer in the network will have. Positive integer.
Following layers will contain a multiple of this number. Lowering this number will likely reduce the amount of
memory required
to train the model.
:param depth: Number of network's levels. Positive integer.
:param loss_function: Loss function also known as cost function. String function name.
:param metrics: Metrics for evaluation of the model performance. List of metrics names.
:param filter_size: sSze of the convolution kernel. Tuple of 3 positive integers.
:param pool_size: Factors by which to downscale (height, width, depth). Tuple of 3 positive integers.
:param concat_axis: Concatenation axis, concatenate over channels. Positive integer.
:param data_format: Ordering of the dimensions in the inputs. Takes values: 'channel_first' or 'channel_last'.
:param padding: Used padding by convolution. Takes values: 'same' or 'valid'.
:param batch_normalization: If set to True, will use Batch Normalization layers after each convolution layer
:param deconvolution: If set to True, will use transpose convolution(deconvolution) instead of up-sampling.
This increases the amount memory required during training.
:return: Compiled untrained 3D brave-net model.
"""
### DOWN-SCALE PATHS
model_inputs = []
down_path_outputs = []
up_path_outputs = []
# store the output of the 2. convolution in the downsampling path to be able to load it in the upsampling path
# for concatenation
down_path_residuals_list = []
# Get low resolution patch size
lri = not input_shapes[0][0] > input_shapes[1][0]
### BUILD SEPARATE DOWN-SAMPLING PATH FOR EACH INPUT
for i, input_shape in enumerate(input_shapes):
# specify the input shape
model_inputs.append(Input(input_shape, name='in-' + str(i)))
current_layer = model_inputs[-1]
residuals = []
# BN for inputs
if batch_normalization:
current_layer = BatchNormalization(axis=concat_axis,
name='bn-' +
str(i))(current_layer)
# scale down input to context path
if i == lri:
current_layer = AveragePooling3D(
pool_size=pool_size)(current_layer)
# build down-sampling path (left side of the brave-net)
# layers on each level: convolution3d -> BN -> convolution3d -> BN -> max-pooling
# last level without max-pooling
for level in range(depth):
conv1 = convolutional_block(
input_layer=current_layer,
n_filters=n_base_filters * (2**level),
activation=activation,
filter_size=filter_size,
padding=padding,
data_format=data_format,
batch_normalization=batch_normalization,
dropout=dropout,
axis=concat_axis,
bn_name='bn-down-input-' + str(i) + '-level-' + str(level) +
'-0',
conv_name='conv-down-input-' + str(i) + '-level-' +
str(level) + '-0')
conv2 = convolutional_block(
input_layer=conv1,
n_filters=n_base_filters * (2**level) * 2,
activation=activation,
filter_size=filter_size,
padding=padding,
data_format=data_format,
batch_normalization=batch_normalization,
dropout=None,
axis=concat_axis,
bn_name='bn-down-input-' + str(i) + '-level-' + str(level) +
'-1',
conv_name='conv-down-input-' + str(i) + '-level-' +
str(level) + '-1')
residuals.append(conv2)
if level < depth - 1:
current_layer = MaxPooling3D(pool_size=pool_size,
data_format=data_format)(conv2)
else:
current_layer = conv2
down_path_outputs.append(current_layer)
down_path_residuals_list.append(residuals)
### BOTTLENECK
# Concatenate feature maps
bottleneck_concat = concatenate(down_path_outputs, axis=concat_axis)
# Fully connected 1
fl1 = Convolution3D(filters=n_base_filters * (2**(depth - 1)) * 2,
kernel_size=(1, 1, 1),
strides=(1, 1, 1),
padding=padding,
data_format=data_format,
activation=activation,
name='conv-fl-0')(bottleneck_concat)
# Fully connected 2
fl2 = Convolution3D(filters=n_base_filters * (2**(depth - 1)) * 2,
kernel_size=(1, 1, 1),
strides=(1, 1, 1),
padding=padding,
data_format=data_format,
activation=activation,
name='conv-fl-1')(fl1)
### UP-SAMPLING PART (right side of the brave-net)
# layers on each level: upconvolution3d -> concatenation with feature maps of corresponding level from down-sampling
# part -> convolution3d -> BN -> convolution3d -> BN
current_layer = fl2
for level in range(depth - 2, -1, -1):
up_scale = upscaling(
input_layer=current_layer,
n_filters=current_layer.shape[-1],
pool_size=pool_size,
padding=padding,
data_format=data_format,
deconvolution=deconvolution) # _keras_shape: output layer shape
# (samples, new_conv_dim1, new_conv_dim2, new_conv_dim3, filters)` if data_format='channels_last'
merged_residuals = concatenate([
down_path_residuals_list[0][level],
down_path_residuals_list[1][level]
],
name='concat-' + str(level) + '-0')
concat = concatenate([merged_residuals, up_scale],
axis=concat_axis,
name='concat-' + str(level) + '-1')
current_layer = convolutional_block(
input_layer=concat,
n_filters=down_path_residuals_list[0][level].shape[-1],
activation=activation,
filter_size=filter_size,
padding=padding,
data_format=data_format,
batch_normalization=batch_normalization,
dropout=dropout,
axis=concat_axis,
bn_name='bn-up-level-' + str(level) + '-0',
conv_name='conv-up-level-' + str(level) + '-0')
current_layer = convolutional_block(
input_layer=current_layer,
n_filters=down_path_residuals_list[0][level].shape[-1],
activation=activation,
filter_size=filter_size,
padding=padding,
data_format=data_format,
batch_normalization=batch_normalization,
dropout=None,
axis=concat_axis,
bn_name='bn-up-level-' + str(level) + '-1',
conv_name='conv-up-level-' + str(level) + '-1')
if 0 < level < depth - 1:
# get level prediction
output = UpSampling3D(
size=(int(input_shapes[1][0] / current_layer.shape[1]),
int(input_shapes[1][1] / current_layer.shape[2]),
int(input_shapes[1][2] / current_layer.shape[3])),
data_format=data_format)(current_layer)
output = Convolution3D(filters=n_classes,
kernel_size=1,
activation=final_activation,
padding=padding,
data_format=data_format,
name='out-' + str(level))(output)
up_path_outputs.append(output)
# final convolutional layer maps feature maps to desired number of classes
if l1 is not None:
if l2 is not None:
final_conv = Convolution3D(filters=n_classes,
kernel_size=1,
activation=final_activation,
padding=padding,
data_format=data_format,
kernel_regularizer=regularizers.l1_l2(
l1=l1, l2=l2),
name='out-0')(current_layer)
else:
final_conv = Convolution3D(filters=n_classes,
kernel_size=1,
activation=final_activation,
padding=padding,
data_format=data_format,
kernel_regularizer=regularizers.l1(l1),
name='out-0')(current_layer)
else:
if l2 is not None:
final_conv = Convolution3D(filters=n_classes,
kernel_size=1,
activation=final_activation,
padding=padding,
data_format=data_format,
kernel_regularizer=regularizers.l2(l2),
name='out-0')(current_layer)
else:
final_conv = Convolution3D(filters=n_classes,
kernel_size=1,
activation=final_activation,
padding=padding,
data_format=data_format,
name='out-0')(current_layer)
up_path_outputs.append(final_conv)
# create the model
model = Model(inputs=model_inputs,
outputs=up_path_outputs,
name='bravenet')
# configure the learning process via the compile function
if not isinstance(metrics, list):
metrics = [metrics]
# Multiple loss
loss, loss_weights = ds_loss(depth=depth, loss_function=loss_function)
model.compile(optimizer=optimizer(lr=learning_rate),
loss=loss,
metrics=metrics,
loss_weights=loss_weights)
print('brave-net compiled.')
# print out model summary to console
model.summary(line_length=120)
return model
def u_net3D(img_height,
img_width,
img_ch,
img_d,
batchNormalization,
Base,
k_size=3):
merge_axis = -1 # Feature maps are concatenated along last axis (for tf backend)
data = Input((img_height, img_width, img_d, img_ch))
#1ConvBlock
conv1 = Convolution3D(padding='same', filters=Base,
kernel_size=k_size)(data)
if batchNormalization:
conv1 = BatchNormalization()(conv1)
conv1 = Activation('relu')(conv1)
conv2 = Convolution3D(padding='same', filters=Base,
kernel_size=k_size)(conv1)
if batchNormalization:
conv2 = BatchNormalization()(conv2)
conv2 = Activation('relu')(conv2)
pool1 = MaxPooling3D(pool_size=(2, 2, 2))(conv2)
#2ConvBlock
conv3 = Convolution3D(padding='same', filters=Base * 2,
kernel_size=k_size)(pool1)
if batchNormalization:
conv3 = BatchNormalization()(conv3)
conv3 = Activation('relu')(conv3)
conv4 = Convolution3D(padding='same', filters=Base * 2,
kernel_size=k_size)(conv3)
if batchNormalization:
conv4 = BatchNormalization()(conv4)
conv4 = Activation('relu')(conv4)
pool2 = MaxPooling3D(pool_size=(2, 2, 2))(conv4)
#3ConvBlock
conv5 = Convolution3D(padding='same', filters=Base * 4,
kernel_size=k_size)(pool2)
if batchNormalization:
conv5 = BatchNormalization()(conv5)
conv5 = Activation('relu')(conv5)
conv6 = Convolution3D(padding='same', filters=Base * 4,
kernel_size=k_size)(conv5)
if batchNormalization:
conv6 = BatchNormalization()(conv6)
conv6 = Activation('relu')(conv6)
pool3 = MaxPooling3D(pool_size=(2, 2, 2))(conv6)
#4ConvBlock
conv7 = Convolution3D(padding='same', filters=Base * 8,
kernel_size=k_size)(pool3)
if batchNormalization:
conv7 = BatchNormalization()(conv7)
conv7 = Activation('relu')(conv7)
conv8 = Convolution3D(padding='same', filters=Base * 8,
kernel_size=k_size)(conv7)
if batchNormalization:
conv8 = BatchNormalization()(conv8)
conv8 = Activation('relu')(conv8)
pool4 = MaxPooling3D(pool_size=(2, 2, 2))(conv8)
#Bottleneck
conv9 = Convolution3D(padding='same',
filters=Base * 16,
kernel_size=k_size)(pool4)
if batchNormalization:
conv9 = BatchNormalization()(conv9)
conv9 = Activation('relu')(conv9)
#Expansion
#1ConvBlock
up1 = Conv3DTranspose(filters=Base * 8,
kernel_size=(2, 2, 2),
padding='same')(conv9)
merged1 = concatenate([up1, conv8], axis=merge_axis)
conv10 = Convolution3D(padding='same',
filters=Base * 8,
kernel_size=k_size)(merged1)
if batchNormalization:
conv10 = BatchNormalization()(conv10)
conv10 = Activation('relu')(conv10)
conv11 = Convolution3D(padding='same',
filters=Base * 8,
kernel_size=k_size)(conv10)
if batchNormalization:
conv11 = BatchNormalization()(conv11)
conv11 = Activation('relu')(conv11)
#2ConvBlock
up2 = Conv3DTranspose(filters=Base * 4,
kernel_size=(2, 2, 2),
padding='same')(conv11)
merged2 = concatenate([up2, conv6], axis=merge_axis)
conv12 = Convolution3D(padding='same',
filters=Base * 4,
kernel_size=k_size)(merged2)
if batchNormalization:
conv12 = BatchNormalization()(conv12)
conv12 = Activation('relu')(conv12)
conv13 = Convolution3D(padding='same',
filters=Base * 4,
kernel_size=k_size)(conv12)
if batchNormalization:
conv13 = BatchNormalization()(conv13)
conv13 = Activation('relu')(conv13)
#3ConvBlock
up3 = Conv3DTranspose(filters=Base * 2,
kernel_size=(2, 2, 2),
padding='same')(conv13)
merged3 = concatenate([up3, conv4], axis=merge_axis)
conv14 = Convolution3D(padding='same',
filters=Base * 2,
kernel_size=k_size)(merged3)
if batchNormalization:
conv14 = BatchNormalization()(conv14)
conv14 = Activation('relu')(conv14)
conv15 = Convolution3D(padding='same', filters=64,
kernel_size=k_size)(conv14)
if batchNormalization:
conv15 = BatchNormalization()(conv15)
conv15 = Activation('relu')(conv15)
#4ConvBlock
up4 = Conv3DTranspose(filters=Base, kernel_size=(2, 2, 2),
padding='same')(conv15)
merged4 = concatenate([up4, conv2], axis=merge_axis)
conv16 = Convolution3D(padding='same', filters=Base,
kernel_size=k_size)(merged4)
if batchNormalization:
conv16 = BatchNormalization()(conv16)
conv16 = Activation('relu')(conv16)
conv17 = Convolution3D(padding='same', filters=Base,
kernel_size=k_size)(conv16)
if batchNormalization:
conv17 = BatchNormalization()(conv17)
conv17 = Activation('relu')(conv17)
#final layer
conv18 = Convolution3D(padding='same', filters=3,
kernel_size=k_size)(merged3)
if batchNormalization:
conv18 = BatchNormalization()(conv18)
output = Reshape([-1, 2])(conv18)
output = Activation('softmax')(output)
output = Reshape(inp_shape[:-1] + (2, ))(output)
model = Model(data, output)
return model
def get_brainseg_3d(input_dim,
num_channels,
dropout,
activation='relu',
final_activation='sigmoid',
kernel_size=(3, 3, 3),
pool_size=(2, 2, 2),
strides=(1, 1, 1),
num_kernels=None,
concat_axis=-1,
data_format='channels_last',
padding='same',
bn=True):
### DOWNS-SCALE PATHS
model_inputs = []
outputs = []
residual_list = []
# --- Get low resolution patch size
lri = 0 if input_dim[0][0] > input_dim[1][0] else 1
hri = 1 if input_dim[0][0] > input_dim[1][0] else 0
### BUILD FEEDFORWARD
for i, dim in enumerate(input_dim):
# specify the input shape
input = Input((dim[0], dim[1], dim[2], num_channels))
model_inputs.append(input)
# BN for inputs
input = BatchNormalization()(input)
# scale down context path
if i == lri:
input = AveragePooling3D(pool_size=(2, 2, 2))(input)
# build down-scale paths
conv, residuals = down_scale_path_3d(input, num_kernels, kernel_size,
strides, pool_size, padding,
activation, dropout, data_format,
bn)
outputs.append(conv)
residual_list.append(residuals)
### BOTTLENECK
# Concat feature maps
concat = concatenate(outputs, axis=-1)
# Fully connected 1
fl1 = Convolution3D(num_kernels[-1], (1, 1, 1),
strides=(1, 1, 1),
padding="same",
activation="relu",
data_format=data_format)(concat)
# Fully connected 2
fl2 = Convolution3D(num_kernels[-1], (1, 1, 1),
strides=(1, 1, 1),
padding="same",
activation="relu",
data_format=data_format)(fl1)
### CONCAT/PREPARE RESIDUALS
merged_residuals = {}
for key in residual_list[0].keys():
merged_residuals[key] = concatenate(
[residual_list[0][key], residual_list[1][key]], axis=-1)
### UP-SCALE PATH
outputs = up_scale_path_ds_3d(fl2, merged_residuals, num_kernels,
kernel_size, strides, pool_size, concat_axis,
padding, activation, final_activation,
dropout, data_format, bn)
# --- Set names to outputs + pretend to use distance input so that save doesnt skip it
for i in range(len(outputs)):
naming_layer = Lambda(lambda x: x[0], name="output-" + str(i))
outputs[i] = naming_layer([outputs[i], model_inputs[0]])
# --- Create model
model = Model(inputs=model_inputs, outputs=outputs)
# --- Print out model summary to console
model.summary()
return model
def get_ds_unet_3d(input_dim,
num_channels,
dropout,
activation='relu',
final_activation='sigmoid',
kernel_size=(3, 3, 3),
pool_size=(2, 2, 2),
strides=(1, 1, 1),
num_kernels=None,
concat_axis=-1,
data_format='channels_last',
padding='same',
bn=True):
"""
Defines the architecture of the u-net. Reconstruction of the u-net introduced in: https://arxiv.org/abs/1505.04597
:param patch_size: height of the patches, positive integer
:param num_channels: number of channels of the input images, positive integer
:param activation: activation_function after every convolution
:param final_activation: activation_function of the final layer
:param optimizer: optimization algorithm for updating the weights and bias values
:param learning_rate: learning_rate of the optimizer, float
:param dropout: percentage of weights to be dropped, float between 0 and 1
:param loss_function: loss function also known as cost function
:param metrics: metrics for evaluation of the model performance
:param kernel_size: size of the convolution kernel, tuple of two positive integers
:param pool_size: factors by which to downscale (vertical, horizontal), tuple of two positive integers
:param strides: strides values, tuple of two positive integers
:param num_kernels: array specifying the number of convolution filters in every level, list of positive integers
containing value for each level of the model
:param concat_axis: concatenation axis, concatenate over channels, positive integer
:param data_format: ordering of the dimensions in the inputs, takes values: 'channel_first' or 'channel_last'
:param padding: used padding by convolution, takes values: 'same' or 'valid'
:param bn: weather to use Batch Normalization layers after each convolution layer, True for use Batch Normalization,
False do not use Batch Normalization
:return: compiled u-net model
"""
# build model
# specify the input shape
inputs = Input((input_dim[0], input_dim[1], input_dim[2], num_channels))
# BN for inputs
layer = BatchNormalization()(inputs)
# --- Down-scale side
conv_down, residuals = down_scale_path_3d(layer, num_kernels, kernel_size,
strides, pool_size, padding,
activation, dropout, data_format,
bn)
# Fully connected 1
fl1 = Convolution3D(num_kernels[-1], (1, 1, 1),
strides=(1, 1, 1),
padding="same",
activation="relu",
data_format=data_format)(conv_down)
# Fully connected 2
fl2 = Convolution3D(num_kernels[-1], (1, 1, 1),
strides=(1, 1, 1),
padding="same",
activation="relu",
data_format=data_format)(fl1)
# --- Up-scale side
outputs = up_scale_path_ds_3d(fl2, residuals, num_kernels, kernel_size,
strides, pool_size, concat_axis, padding,
activation, final_activation, dropout,
data_format, bn)
# --- Set names to outputs
for i in range(len(outputs)):
naming_layer = Lambda(lambda x: x[0], name="output-" + str(i))
outputs[i] = naming_layer([outputs[i], inputs])
model = Model(inputs=inputs, outputs=outputs)
# print out model summary to console
model.summary()
return model
