def vae(model_path, num_channels, dims, learning_rate):
intermediate_dim = 64
latent_dim = 2
epsilon_std = 1.0
# full vae
input_img = Input(shape=dims)
x = Conv3D(64, (3, 3, 3), activation='relu', padding='same')(input_img)
x = MaxPooling3D((2, 2, 2), padding='same')(x)
x = Conv3D(64, (3, 3, 3), activation='relu', padding='same')(x)
x = MaxPooling3D((2, 2, 2), padding='same')(x)
x = Conv3D(64, (3, 3, 3), activation='relu', padding='same')(x)
x = MaxPooling3D((2, 2, 2), padding='same')(x)
x = Conv3D(64, (3, 3, 3), activation='relu', padding='same')(x)
x = MaxPooling3D((2, 2, 2), padding='same')(x)
x = Conv3D(64, (3, 3, 3), activation='relu', padding='same')(x)
x = MaxPooling3D((2, 2, 2), padding='same')(x)
flat = Flatten()(x)
hidden = Dense(intermediate_dim, activation='relu')(flat)
z_mean = Dense(latent_dim)(hidden)
z_log_var = Dense(latent_dim)(hidden)
def sampling(args):
z_mean, z_log_var = args
epsilon = K.random_normal(shape=(K.shape(z_mean)[0], latent_dim),
mean=0.,
stddev=epsilon_std)
return z_mean + K.exp(z_log_var) * epsilon
z = Lambda(sampling)([z_mean, z_log_var])
x = Dense(intermediate_dim, activation='relu')(z)
x = Dense(3 * 3 * 4 * 64, activation='relu')(x)
x = Reshape(target_shape=(3, 3, 4, 64))(x)
x = UpSampling3D((2, 2, 2))(x)
x = Conv3D(64, (3, 3, 3), activation='relu', padding='same')(x)
x = UpSampling3D((2, 2, 2))(x)
x = Conv3D(64, (3, 3, 3), activation='relu', padding='same')(x)
x = UpSampling3D((2, 2, 2))(x)
x = Conv3D(64, (3, 3, 3), activation='relu', padding='same')(x)
x = UpSampling3D((2, 2, 2))(x)
x = Conv3D(64, (3, 3, 3), activation='relu', padding='same')(x)
x = UpSampling3D((2, 2, 2))(x)
decoded = Conv3D(num_channels, (3, 3, 3),
activation='linear',
padding='same')(x)
vae = Model(input_img, decoded)
def mae_loss(y_true, y_pred):
return metrics.mean_absolute_error(K.flatten(y_true),
K.flatten(y_pred))
def kl_loss(y_true, y_pred):
return -0.5 * K.sum(
1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
def disentangled_kl_loss(y_true, y_pred):
beta = 0.1
kl_loss = -0.5 * K.sum(
1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return beta * kl_loss
def vae_loss(y_true, y_pred):
return K.mean(
mae_loss(y_true, y_pred) + disentangled_kl_loss(y_true, y_pred))
vae.compile(
optimizer=Adam(lr=learning_rate),
loss=vae_loss,
metrics=[mae_loss, kl_loss, disentangled_kl_loss],
)
print(vae.summary())
json_string = vae.to_json()
with open(model_path, 'w') as f:
json.dump(json_string, f)
return vae
def create_model(self, channel_size, row_size, n_filter, filter_size, lr,
TIME_POINT, *unused_params):
main_input = Input(shape=(TIME_POINT, channel_size, row_size,
row_size),
dtype='float32',
name='main_input')
conv1 = NConvolution3D(n_filter, 3, 3, 3)(main_input)
conv1 = NConvolution3D(n_filter, 1, 1, 1)(conv1)
pool1 = MaxPooling3D(pool_size=(2, 2, 2))(conv1)
pool1 = Dropout_uncertain(DROP_RATE)(pool1)
conv2 = NConvolution3D(2 * n_filter, 3, 3, 3)(pool1)
conv2 = NConvolution3D(2 * n_filter, 1, 1, 1)(conv2)
pool2 = MaxPooling3D(pool_size=(2, 2, 2))(conv2)
pool2 = Dropout_uncertain(DROP_RATE)(pool2)
conv3 = NConvolution3D(3 * n_filter, 3, 3, 3)(pool2)
conv3 = NConvolution3D(3 * n_filter, 1, 1, 1)(conv3)
conv3 = Dropout_uncertain(DROP_RATE)(conv3)
up4 = concatenate([UpSampling3D(size=(2, 2, 2))(conv3), conv2], axis=1)
up4 = NConvolution3D(2 * n_filter, 1, 1, 1)(up4)
conv4 = NConvolution3D(2 * n_filter, 3, 3, 3)(up4)
conv4 = Dropout_uncertain(DROP_RATE)(conv4)
#Scaled Part
scaled_input = Input(shape=(TIME_POINT, channel_size, row_size,
row_size),
dtype='float32',
name='aug_input')
scaled_conv1 = NConvolution3D(n_filter, 3, 3, 3)(scaled_input)
scaled_conv1 = NConvolution3D(n_filter, 1, 1, 1)(scaled_conv1)
scaled_pool1 = MaxPooling3D(pool_size=(2, 2, 2))(scaled_conv1)
scaled_pool1 = Dropout_uncertain(DROP_RATE)(scaled_pool1)
scaled_conv2 = NConvolution3D(2 * n_filter, 3, 3, 3)(scaled_pool1)
scaled_conv2 = NConvolution3D(2 * n_filter, 1, 1, 1)(scaled_conv2)
scaled_pool2 = MaxPooling3D(pool_size=(2, 2, 2))(scaled_conv2)
scaled_pool2 = Dropout_uncertain(DROP_RATE)(scaled_pool2)
scaled_conv3 = NConvolution3D(3 * n_filter, 3, 3, 3)(scaled_pool2)
scaled_conv3 = NConvolution3D(3 * n_filter, 1, 1, 1)(scaled_conv3)
scaled_conv3 = Dropout_uncertain(DROP_RATE)(scaled_conv3)
scaled_up4 = concatenate(
[UpSampling3D(size=(2, 2, 2))(scaled_conv3), scaled_conv2], axis=1)
scaled_up4 = NConvolution3D(2 * n_filter, 1, 1, 1)(scaled_up4)
scaled_conv4 = NConvolution3D(2 * n_filter, 3, 3, 3)(scaled_up4)
scaled_conv4 = Dropout_uncertain(DROP_RATE)(scaled_conv4)
#Merge two parts
up5 = concatenate([UpSampling3D(size=(2, 2, 2))(conv4), conv1, \
UpSampling3D(size=(2, 2, 2))(scaled_conv4), scaled_conv1], axis=1)
conv5 = NConvolution3D(3 * n_filter, 1, 1, 1)(up5)
conv5 = NConvolution3D(3 * n_filter, 3, 3, 3)(conv5)
conv5 = Dropout_uncertain(DROP_RATE)(conv5)
conv6 = NConvolution3D(2 * n_filter, 3, 3, 3)(conv5)
conv6 = Dropout_uncertain(DROP_RATE)(conv6)
# sampling normal
z_avg = Conv3D(1, (1, 3, 3), padding='same',
activation='linear')(conv6)
z_log_var = Conv3D(1, (1, 3, 3), padding='same',
activation='linear')(conv6)
z = SampleNormal()([z_avg, z_log_var])
# conv7=Activation('sigmoid')(z)
conv7 = z
model = Model(inputs=[main_input, scaled_input], outputs=conv7)
"""
model.compile(optimizer=Adam(lr=lr), \
loss=dice_coef_loss, metrics=[dice_coef, 'binary_accuracy', recall_smooth])
"""
model.compile(optimizer=Adam(lr=lr), \
loss=stable_nll_loss, metrics=[stable_dice_coef, stable_recall_smooth])
return model
def get_vnet_gen_model():
inputs = Input((1, num_slices, img_height, img_width))
# output 128 x 128 x 16 x 16
conv1 = Conv3D(16, (5, 5, 5), activation='relu',
border_mode='same')(inputs)
# output 128 x 128 x 16 x 32
m1 = merge([inputs, conv1], mode='concat', concat_axis=1)
#m1 = add([inputs, conv1])
# output 64 x 64 x 8 x 32
pool1 = MaxPooling3D(pool_size=(2, 2, 2))(m1)
# output 64 x 64 x 8 x 32
conv2 = Conv3D(32, (5, 5, 5), activation='relu', border_mode='same')(pool1)
# output 64 x 64 x 8 x 32
conv2 = Conv3D(32, 5, 5, 5, activation='relu', border_mode='same')(conv2)
# output 64 x 64 x 8 x 64
m2 = merge([pool1, conv2], mode='concat', concat_axis=1)
# output 32 x 32 x 4 x 32
pool2 = MaxPooling3D(pool_size=(2, 2, 2))(m2)
# output 32 x 32 x 4 x 64
conv3 = Conv3D(64, 5, 5, 5, activation='relu', border_mode='same')(pool2)
# output 32 x 32 x 4 x 64
conv3 = Conv3D(64, 5, 5, 5, activation='relu', border_mode='same')(conv3)
# output 32 x 32 x 4 x 64
conv3 = Conv3D(64, 5, 5, 5, activation='relu', border_mode='same')(conv3)
# output 32 x 32 x 4 x 96
m3 = merge([pool2, conv3], mode='concat', concat_axis=1)
# output 16 x 16 x 2 x 96
pool3 = MaxPooling3D(pool_size=(2, 2, 2))(m3)
conv4 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(pool3)
conv4 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(conv4)
# output 16 x 16 x 2 x 128
conv4 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(conv4)
# output 16 x 16 x 2 x 224
m4 = merge([pool3, conv4], mode='concat', concat_axis=1)
# output 8 x 8 x 1 x 224
pool4 = MaxPooling3D(pool_size=(2, 2, 2))(m4)
conv5 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(pool4)
conv5 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(conv5)
# output 8 x 8 x 1 x 256
conv5 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(conv5)
# output 8 x 8 x 1 x 480
m5 = merge([pool4, conv5], mode='concat', concat_axis=1)
# output 16 x 16 x 2 x 480
up5 = UpSampling3D(size=(2, 2, 2))(m5)
conv6 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(up5)
conv6 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(conv6)
# output 16 x 16 x 2 x 256
conv6 = Conv3D(256, 5, 5, 5, activation='relu', border_mode='same')(conv6)
# output 16 x 16 x 2 x 736
m6 = merge([up5, conv6], mode='concat', concat_axis=1)
# output 32 x 32 x 4 x 736
up6 = UpSampling3D(size=(2, 2, 2))(m6)
conv7 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(up6)
conv7 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(conv7)
# output 32 x 32 x 4 x 128
conv7 = Conv3D(128, 5, 5, 5, activation='relu', border_mode='same')(conv7)
# output 32 x 32 x 4 x 864
m7 = merge([up6, conv7], mode='concat', concat_axis=1)
# output 64 x 64 x 8 x 864
up7 = UpSampling3D(size=(2, 2, 2))(m7)
conv8 = Conv3D(64, 5, 5, 5, activation='relu', border_mode='same')(up7)
# output 64 x 64 x 8 x 64
conv8 = Conv3D(64, 5, 5, 5, activation='relu', border_mode='same')(conv8)
# output 64 x 64 x 8 x 928
m8 = merge([up7, conv8], mode='concat', concat_axis=1)
# output 128 x 128 x 16 x 928
up8 = UpSampling3D(size=(2, 2, 2))(m8)
# output 128 x 128 x 16 x 32
conv9 = Conv3D(32, 5, 5, 5, activation='relu', border_mode='same')(up8)
# output 128 x 128 x 16 x 960
m9 = merge([up8, conv9], mode='concat', concat_axis=1)
conv10 = Conv3D(1, 1, 1, 1, activation='sigmoid', border_mode='same')(m9)
model = Model(input=inputs, output=conv10)
model.compile(optimizer=Adam(lr=1.0e-5),
loss=dice_coef_loss,
metrics=[dice_coef])
model.summary()
return model
def createModel(self):
'''Creates model architecture
Input: Data input dimensions, eventually architecture specifications parsed from a config file? (activations, costFunction, hyperparameters (nr layers), dropout....)
Output: Keras Model'''
#seed = 1337
#mod1 = Input((self.dpatch,self.dpatch,self.dpatch, self.num_channels))
mod1 = Input((None, None, None,
self.num_channels)) # last channel is the individual TPM
############# Normal pathway ##################
# reduces 57 into 9 ( - 48)
x1 = Cropping3D(cropping=((16, 16), (16, 16), (16, 16)),
input_shape=(None, None, None,
self.num_channels))(mod1)
# 25 , to 9 = -16
for feature in self.conv_features:
x1 = Conv3D(
filters=feature,
kernel_size=(3, 3, 3),
#kernel_initializer=he_normal(seed=seed),
kernel_initializer=he_normal(),
kernel_regularizer=regularizers.l2(self.L2))(x1)
x1 = LeakyReLU()(x1)
x1 = BatchNormalization()(x1)
############# Downsampled pathway ##################
#x2 = MaxPooling3D(pool_size=(self.d_factor,self.d_factor,self.d_factor), padding="same")(mod1)
x2 = AveragePooling3D(pool_size=(self.d_factor, self.d_factor,
self.d_factor),
padding="same")(mod1)
# Reduces into by 1/3 = 19, then down to 3 : -16
for feature in self.conv_features:
x2 = Conv3D(
filters=feature,
kernel_size=(3, 3, 3),
#kernel_initializer=he_normal(seed=seed),
kernel_initializer=he_normal(),
kernel_regularizer=regularizers.l2(self.L2))(x2)
x2 = LeakyReLU()(x2)
x2 = BatchNormalization()(x2)
x2 = UpSampling3D(size=(3, 3, 3))(x2)
############# Fully connected layers ##################
x = concatenate([x1, x2])
x = Conv3D(
filters=self.fc_features[0],
kernel_size=(1, 1, 1),
#kernel_initializer=he_normal(seed=seed),
kernel_initializer=Orthogonal(),
kernel_regularizer=regularizers.l2(self.L2))(x)
x = LeakyReLU()(x)
x = BatchNormalization()(x)
x = Conv3D(
filters=self.fc_features[1],
kernel_size=(1, 1, 1),
#kernel_initializer=he_normal(seed=seed),
kernel_initializer=Orthogonal(),
kernel_regularizer=regularizers.l2(self.L2))(x)
x = LeakyReLU()(x)
x = BatchNormalization()(x)
#
# x = Conv3D(filters = self.output_classes,
# kernel_size = (1,1,1),
# #kernel_initializer=he_normal(seed=seed),
# kernel_initializer=Orthogonal(),
# kernel_regularizer=regularizers.l2(self.L2))(x)
# #x = BatchNormalization()(x)
# tpm = Input((None,None,None,6))
#x4 = Cropping3D(cropping = ((24,24),(24,24),(24,24)), input_shape=(None, None, None, self.num_channels))(mod1)
#x = concatenate([x,tpm])#,x4]) # MIXING ONLY CHANNELS + CHANNELS.
# Skipping this bandfilter and going straigth to the softmax makes everything pointless (no nonlinearity besides softmax), and pushes performance to the floor.
# x = Conv3D(filters = self.fc_features[1],
# kernel_size = (1,1,1),
# kernel_initializer=Orthogonal(),
# kernel_regularizer=regularizers.l2(self.L2))(x)
# x = LeakyReLU()(x)
# x = BatchNormalization()(x)
#
# x = Conv3D(filters = self.fc_features[1],
# kernel_size = (1,1,1),
# kernel_initializer=Orthogonal(),
# kernel_regularizer=regularizers.l2(self.L2))(x)
# x = LeakyReLU()(x)
x = Conv3D(filters=self.output_classes,
kernel_size=(1, 1, 1),
kernel_initializer=Orthogonal(),
kernel_regularizer=regularizers.l2(self.L2))(x)
x = Activation(softmax)(x)
model = Model(inputs=[mod1], outputs=x)
#print_summary(model, positions=[.33, .6, .67,1])
#rmsprop = RMSprop(lr=self.learning_rate, rho=0.9, epsilon=1e-8, decay=self.optimizer_decay)
if self.loss_function == 'Multinomial':
model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=self.learning_rate),
metrics=[
dice_coef_multilabel0, dice_coef_multilabel1,
dice_coef_multilabel2, dice_coef_multilabel3,
dice_coef_multilabel4, dice_coef_multilabel5,
dice_coef_multilabel6
])
elif self.loss_function == 'Dice2':
model.compile(
loss=Generalised_dice_coef_multilabel2,
optimizer=Adam(lr=self.learning_rate),
metrics=[dice_coef_multilabel0, dice_coef_multilabel1])
elif self.loss_function == 'Dice6':
model.compile(loss=dice_coef_multilabel6,
optimizer=Adam(lr=self.learning_rate),
metrics=[
dice_coef_multilabel0, dice_coef_multilabel1,
dice_coef_multilabel2, dice_coef_multilabel3,
dice_coef_multilabel4, dice_coef_multilabel5
])
elif self.loss_function == 'wDice6':
model.compile(loss=w_dice_coef_multilabel6,
optimizer=Adam(lr=self.learning_rate),
metrics=[
dice_coef_multilabel0, dice_coef_multilabel1,
dice_coef_multilabel2, dice_coef_multilabel3,
dice_coef_multilabel4, dice_coef_multilabel5
])
elif self.loss_function == 'wDice2':
model.compile(
loss=w_dice_coef_multilabel2,
optimizer=Adam(lr=self.learning_rate),
metrics=[dice_coef_multilabel0, dice_coef_multilabel1])
elif self.loss_function == 'Dice7':
model.compile(loss=Generalised_dice_coef_multilabel7,
optimizer=Adam(lr=self.learning_rate),
metrics=[
dice_coef_multilabel0, dice_coef_multilabel1,
dice_coef_multilabel2, dice_coef_multilabel3,
dice_coef_multilabel4, dice_coef_multilabel5,
dice_coef_multilabel6
])
return model
#dm = MultiPriors_noTPM_Model(7, 1, 0.001, [0], 0.01, 0, 'Dice7' )
#model = dm.createModel()
#model.summary()
#from keras.utils import plot_model
#plot_model(model, to_file='/home/hirsch/Documents/projects/brainSegmentation/DeepPriors' +'/multiscale_TPM_noTPM.png', show_shapes=True)
#
##
#X = np.random.randn(1,57,57,57,1)
#y = np.random.binomial(n=1, p=0.5,size=9**3*7).reshape(1,9,9,9,7)
#y.shape
#
#TPM = np.random.randn(1,9,9,9,6)
#
#yhat = model.predict([X,TPM])
#yhat.shape
#
#model.fit([X,TPM], y)
def isensee2017_3D(n_labels,shape,W,lr=1e-5, n_base_filters=16, depth=4, dropout_rate=0.3,
n_segmentation_levels=3, optimizer=Adam, initial_learning_rate=9e-4,
loss_function=weighted_dice_coefficient_loss, activation_name="sigmoid"):
"""
This function builds a model proposed by Isensee et al. for the BRATS 2017 competition:
https://www.cbica.upenn.edu/sbia/Spyridon.Bakas/MICCAI_BraTS/MICCAI_BraTS_2017_proceedings_shortPapers.pdf
This network is highly similar to the model proposed by Kayalibay et al. "CNN-based Segmentation of Medical
Imaging Data", 2017: https://arxiv.org/pdf/1701.03056.pdf
:param input_shape:
:param n_base_filters:
:param depth:
:param dropout_rate:
:param n_segmentation_levels:
:param n_labels:
:param optimizer:
:param initial_learning_rate:
:param loss_function:
:param activation_name:
:return:
"""
inputs = Input(shape)
current_layer = inputs
level_output_layers = list()
level_filters = list()
for level_number in range(depth):
n_level_filters = (2**level_number) * n_base_filters
level_filters.append(n_level_filters)
if current_layer is inputs:
in_conv = create_convolution_block(current_layer, n_level_filters,activation=LeakyReLU, instance_normalization=False)
else:
in_conv = create_convolution_block(current_layer, n_level_filters, strides=(2, 2, 2),activation=LeakyReLU, instance_normalization=False)
context_output_layer = create_context_module(in_conv, n_level_filters, dropout_rate=dropout_rate)
summation_layer = Add()([in_conv, context_output_layer])
level_output_layers.append(summation_layer)
current_layer = summation_layer
segmentation_layers = list()
for level_number in range(depth - 2, -1, -1):
up_sampling = create_up_sampling_module(current_layer, level_filters[level_number])
concatenation_layer = concatenate([level_output_layers[level_number], up_sampling], axis=4)
localization_output = create_localization_module(concatenation_layer, level_filters[level_number])
current_layer = localization_output
if level_number < n_segmentation_levels:
segmentation_layers.insert(0, create_convolution_block(current_layer, n_filters=n_labels, kernel=(1, 1, 1)))
# output_layer = current_layer
output_layer = None
for level_number in reversed(range(n_segmentation_levels)):
segmentation_layer = segmentation_layers[level_number]
if output_layer is None:
output_layer = segmentation_layer
else:
output_layer = Add()([output_layer, segmentation_layer])
if level_number > 0:
output_layer = UpSampling3D(size=(2, 2, 2))(output_layer)
activation_block = Activation(activation_name)(output_layer)
if n_labels==1:
activation_block = Activation('sigmoid')(output_layer)
if n_labels>1:
final_convolution = Conv3D(n_labels, 1)(output_layer)
o = Reshape((shape[0] * shape[1]* shape[2],n_labels), input_shape=(shape[0], shape[1], shape[2],n_labels))(final_convolution)
activation_block = Activation('softmax')(o)
model = Model(inputs=inputs, outputs=activation_block)
if W !='':
model.load_weights(W)
if n_labels == 1:
model.compile(loss='binary_crossentropy',optimizer=Adam(lr=lr),metrics=['accuracy'])
if n_labels > 1:
model.compile(loss='categorical_crossentropy',optimizer=Adam(lr=lr),metrics=['categorical_accuracy'])
model.summary()
return model
pool1 = MaxPooling3D(pool_size=(2, 2, 1))(conv1) conv2 = Conv3D(8, (3, 3, 3), **conv_properties)(pool1) pool2 = MaxPooling3D(pool_size=(2, 2, 1))(conv2) conv3 = Conv3D(4, (3, 3, 1), **conv_properties)(pool2) pool3 = MaxPooling3D(pool_size=(2, 2, 1))(conv3) conv4 = Conv3D(4, (3, 3, 1), **conv_properties)(pool3) drop4 = Dropout(0.2)(conv4) pool4 = MaxPooling3D(pool_size=(2, 2, 1))(drop4) conv5 = Conv3D(4, (3, 3, 1), **conv_properties)(pool4) drop5 = Dropout(0.2)(conv5) up6 = UpSampling3D(size=(2, 2, 1))(drop5) up6 = Conv3D(4, (2, 2, 1), **conv_properties)(up6) cat6 = Concatenate(axis=-1)([drop4, up6]) conv6 = Conv3D(4, (3, 3, 1), **conv_properties)(cat6) up7 = UpSampling3D(size=(2, 2, 1))(conv6) up7 = Conv3D(4, (2, 2, 1), **conv_properties)(up7) cat7 = Concatenate(axis=-1)([conv3, up7]) conv7 = Conv3D(4, (3, 3, 1), **conv_properties)(cat7) up8 = UpSampling3D(size=(2, 2, 1))(conv7) up8 = Conv3D(8, (2, 2, 1), **conv_properties)(up8) cat8 = Concatenate(axis=-1)([conv2, up8]) conv8 = Conv3D(8, 3, **conv_properties)(cat8) up9 = UpSampling3D(size=(2, 2, 1))(conv8)
def get_unet_3d(self):
inputs = Input((self.img_x, self.img_y, self.img_z, 1))
# inp_norm = BatchNormalization()(inputs)
conv1 = Conv3D(32, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal')(inputs)
conv1 = Conv3D(32, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv1)
pool1 = MaxPooling3D(pool_size=(2, 2, 2))(conv1)
conv2 = Conv3D(64, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool1)
conv2 = Conv3D(64, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv2)
pool2 = MaxPooling3D(pool_size=(2, 2, 2))(conv2)
conv3 = Conv3D(128, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool2)
conv3 = Conv3D(128, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv3)
pool3 = MaxPooling3D(pool_size=(2, 2, 2))(conv3)
'''
conv4 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer = 'he_normal')(pool3)
conv4 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer = 'he_normal')(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)
'''
conv5 = Conv3D(256, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool3)
conv5 = Conv3D(256, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv5)
'''
up6 = concatenate([UpSampling2D((2, 2))(conv5), conv4], axis=3)
conv6 = Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer = 'he_normal')(up6)
conv6 = Conv2D(256, (3, 3), activation='relu', padding='same')(conv6)
'''
up7 = concatenate([UpSampling3D((2, 2, 2))(conv5), conv3], axis=4)
conv7 = Conv3D(128, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal')(up7)
conv7 = Conv3D(128, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv7)
up8 = concatenate([UpSampling3D((2, 2, 2))(conv7), conv2], axis=4)
conv8 = Conv3D(64, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal')(up8)
conv8 = Conv3D(64, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv8)
up9 = concatenate([UpSampling3D((2, 2, 2))(conv8), conv1], axis=4)
conv9 = Conv3D(32, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal')(up9)
conv9 = Conv3D(32, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv9)
conv10 = Conv3D(self.label_num, (1, 1, 1), activation='softmax')(conv9)
model = Model(inputs=[inputs], outputs=[conv10])
model.compile(optimizer=Adam(lr=1e-4),
loss=dice_coef_loss_weighted,
metrics=[dice_coef_weighted])
# default lr=1e-4
self.model = model
return model
def unet(input_size=None, label_nums=2):
if input_size is None:
input_size = (64, 64, 64, 1)
inputs = Input(input_size)
conv1 = Conv3D(16,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(inputs)
conv1 = Conv3D(32,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(conv1)
pool1 = MaxPooling3D(pool_size=(2, 2, 2),
data_format="channels_last")(conv1)
conv2 = Conv3D(32,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(pool1)
conv2 = Conv3D(64,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(conv2)
pool2 = MaxPooling3D(pool_size=(2, 2, 2),
data_format="channels_last")(conv2)
conv3 = Conv3D(64,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(pool2)
conv3 = Conv3D(128,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(conv3)
pool3 = MaxPooling3D(pool_size=(2, 2, 2),
data_format="channels_last")(conv3)
conv4 = Conv3D(128,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(pool3)
conv4 = Conv3D(256,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(conv4)
pool4 = MaxPooling3D(pool_size=(2, 2, 2),
data_format="channels_last")(conv4)
conv5 = Conv3D(256,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(pool4)
conv5 = Conv3D(512,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(conv5)
up1 = UpSampling3D(size=(2, 2, 2), data_format="channels_last")(conv5)
up1 = concatenate([conv4, up1], axis=-1)
conv6 = Conv3D(256,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(up1)
conv6 = Conv3D(256,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(conv6)
up2 = UpSampling3D(size=(2, 2, 2), data_format="channels_last")(conv6)
up2 = concatenate([conv3, up2], axis=-1)
conv7 = Conv3D(128,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(up2)
conv7 = Conv3D(128,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(conv7)
up3 = UpSampling3D(size=(2, 2, 2), data_format="channels_last")(conv7)
up3 = concatenate([conv2, up3], axis=-1)
conv8 = Conv3D(64,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(up3)
conv8 = Conv3D(64,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(conv8)
up4 = UpSampling3D(size=(2, 2, 2), data_format="channels_last")(conv8)
up4 = concatenate([conv1, up4], axis=-1)
conv9 = Conv3D(32,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(up4)
conv9 = Conv3D(32,
3,
activation='relu',
padding='same',
data_format="channels_last",
kernel_initializer='he_normal')(conv9)
conv10 = Conv3D(label_nums, 1, activation='softmax')(conv9)
return Model(input=inputs, output=conv10)
def get_unet3d_gen_model():
inputs = Input((1, num_slices, img_height, img_width))
# output W2 = 256 x 256 x 16 x 32
conv1 = Convolution3D(32, 3, 3, 3, activation='relu',
border_mode='same')(inputs)
# output 256 x 256 x 16 x 32
conv1 = Convolution3D(32, 3, 3, 3, activation='relu',
border_mode='same')(conv1)
# output 128 x 128 x 8 x 32
pool1 = MaxPooling3D(pool_size=(2, 2, 2))(conv1)
# output 128 x 128 x 8 x 64
conv2 = Convolution3D(64, 3, 3, 3, activation='relu',
border_mode='same')(pool1)
# output 128 x 128 x 8 x 64
conv2 = Convolution3D(64, 3, 3, 3, activation='relu',
border_mode='same')(conv2)
# output 64 x 64 x 4 x 64
pool2 = MaxPooling3D(pool_size=(2, 2, 2))(conv2)
# output 64 x 64 x 4 x 128
conv3 = Convolution3D(128, 3, 3, 3, activation='relu',
border_mode='same')(pool2)
#output 64 x 64 x 4 x 128
conv3 = Convolution3D(128, 3, 3, 3, activation='relu',
border_mode='same')(conv3)
# output 32 x 32 x 2 x 128
pool3 = MaxPooling3D(pool_size=(2, 2, 2))(conv3)
# output 32 x 32 x 2 x 256
conv4 = Convolution3D(256, 3, 3, 3, activation='relu',
border_mode='same')(pool3)
# output 32 x 32 x 2 x 256
conv4 = Convolution3D(256, 3, 3, 3, activation='relu',
border_mode='same')(conv4)
# output 16 x 16 x 1 x 256
pool4 = MaxPooling3D(pool_size=(2, 2, 2))(conv4)
# output 16 x 16 x 1 x 512
conv5 = Convolution3D(512, 3, 3, 3, activation='relu',
border_mode='same')(pool4)
# output 16 x 16 x 1 x 512
conv5 = Convolution3D(512, 3, 3, 3, activation='relu',
border_mode='same')(conv5)
# output of UpSampling2D(size=(2, 2, 2))(conv5): 32 32 x 2 x 512
# outputof up6: 32 x 32 x 2 x 768
up6 = merge([UpSampling3D(size=(2, 2, 2))(conv5), conv4],
mode='concat',
concat_axis=1)
# output 32 x 32 x 2 x 256
conv6 = Convolution3D(256, 3, 3, 3, activation='relu',
border_mode='same')(up6)
#conv6 = Convolution3D(256, 3, 3, 3, activation='relu', border_mode='same')(pool3)
# output 32 x 32 x 2 x 256
conv6 = Convolution3D(256, 3, 3, 3, activation='relu',
border_mode='same')(conv6)
# output of UpSampling2D(size=(2, 2, 2))(conv6): 64 x 64 x 4x 256
# outputof up7: 64 x 64 x 4 x 384
up7 = merge([UpSampling3D(size=(2, 2, 2))(conv6), conv3],
mode='concat',
concat_axis=1)
# output 64 x 64 x 4 x 128
conv7 = Convolution3D(128, 3, 3, 3, activation='relu',
border_mode='same')(up7)
# output 64 x 64 x 4 x 128
conv7 = Convolution3D(128, 3, 3, 3, activation='relu',
border_mode='same')(conv7)
# output of UpSampling2D(size=(2, 2, 2))(conv7): 128 x 128 x 8 x 128
# outputof up8: 128 x 128 x 8 x 192
up8 = merge([UpSampling3D(size=(2, 2, 2))(conv7), conv2],
mode='concat',
concat_axis=1)
# output 128 x 128 x 8 x 64
conv8 = Convolution3D(64, 3, 3, 3, activation='relu',
border_mode='same')(up8)
# output 128 x 128 x 8 x 64
conv8 = Convolution3D(64, 3, 3, 3, activation='relu',
border_mode='same')(conv8)
# output of UpSampling2D(size=(2, 2, 2))(conv8): 256 x 256 x 16 x 64
# outputof up9: 256 x 256 x 16 x 96
up9 = merge([UpSampling3D(size=(2, 2, 2))(conv8), conv1],
mode='concat',
concat_axis=1)
# output 256 x 256 x 16 x 32
conv9 = Convolution3D(32, 3, 3, 3, activation='relu',
border_mode='same')(up9)
# output 256 x 256 x 16 x 32
conv9 = Convolution3D(32, 3, 3, 3, activation='relu',
border_mode='same')(conv9)
# output 256 x 256 x 16 x 1
conv10 = Convolution3D(1, 1, 1, 1, activation='sigmoid')(conv9)
model = Model(input=inputs, output=conv10)
model.compile(optimizer=Adam(lr=1.0e-5),
loss=dice_coef_loss,
metrics=[dice_coef])
model.summary()
return model
def isensee2017_model(input_shape=(4, 128, 128, 128),
n_base_filters=16,
depth=5,
dropout_rate=0.3,
n_segmentation_levels=3,
n_labels=4,
activation_name="sigmoid"):
"""
This function builds a model proposed by Isensee et al. for the BRATS 2017 competition:
https://www.cbica.upenn.edu/sbia/Spyridon.Bakas/MICCAI_BraTS/MICCAI_BraTS_2017_proceedings_shortPapers.pdf
This network is highly similar to the model proposed by Kayalibay et al. "CNN-based Segmentation of Medical
Imaging Data", 2017: https://arxiv.org/pdf/1701.03056.pdf
:param input_shape:
:param n_base_filters:
:param depth:
:param dropout_rate:
:param n_segmentation_levels:
:param n_labels:
:param optimizer:
:param initial_learning_rate:
:param loss_function:
:param activation_name:
:return:
"""
inputs = Input(input_shape)
current_layer = inputs
level_output_layers = list(
) # result of each level after doing summation of residual
level_filters = list() # number of filters regards each level
for level_number in range(depth):
n_level_filters = (2**level_number) * n_base_filters
level_filters.append(n_level_filters)
if current_layer is inputs:
# level 0 (up-most) of left path does not stride
in_conv = create_convolution_block(current_layer, n_level_filters)
else:
# other levels of left path stride 2
in_conv = create_convolution_block(current_layer,
n_level_filters,
strides=(2, 2, 2))
# after 3x3x3 conv (and stride 2x2x2), pass through context modules
context_output_layer = create_context_module(in_conv,
n_level_filters,
dropout_rate=dropout_rate)
# using result of context module and result of conv 3x3x3 to perform residual learning
summation_layer = Add()([in_conv, context_output_layer])
# position on left path before passing on "-- line" to get to right path
level_output_layers.append(summation_layer)
# at the end of the for loop, "current_layer" will be level 4 (bottom-most one)
current_layer = summation_layer
segmentation_layers = list()
for level_number in range(depth - 2, -1, -1):
# start upsampling (from level 4)
# the first iter: upsampling result of level 4 to scale of level 3
up_sampling = create_up_sampling_module(current_layer,
level_filters[level_number])
concatenation_layer = concatenate(
[level_output_layers[level_number], up_sampling], axis=1)
localization_output = create_localization_module(
concatenation_layer, level_filters[level_number])
current_layer = localization_output
if level_number < n_segmentation_levels:
segmentation_layers.insert(
0,
Conv3D(n_labels, (1, 1, 1))(current_layer))
output_layer = None
for level_number in reversed(range(n_segmentation_levels)):
segmentation_layer = segmentation_layers[level_number]
if output_layer is None:
output_layer = segmentation_layer
else:
output_layer = Add()([output_layer, segmentation_layer])
if level_number > 0:
output_layer = UpSampling3D(size=(2, 2, 2))(output_layer)
activation_block = None
if activation_name == 'sigmoid':
activation_block = Activation(activation_name)(output_layer)
elif activation_name == 'softmax':
activation_block = Softmax(axis=1)(output_layer)
model = Model(inputs=inputs, outputs=activation_block)
return model
def isensee2017_model(input_shape=(4, 128, 128, 128),
n_base_filters=16,
depth=5,
dropout_rate=0.3,
n_segmentation_levels=3,
n_labels=4,
optimizer=Adam,
initial_learning_rate=5e-4,
loss_function=weighted_dice_coefficient_loss,
activation_name="sigmoid"):
"""
This function builds a model proposed by Isensee et al. for the BRATS 2017 competition:
https://www.cbica.upenn.edu/sbia/Spyridon.Bakas/MICCAI_BraTS/MICCAI_BraTS_2017_proceedings_shortPapers.pdf
This network is highly similar to the model proposed by Kayalibay et al. "CNN-based Segmentation of Medical
Imaging Data", 2017: https://arxiv.org/pdf/1701.03056.pdf
:param input_shape:
:param n_base_filters:
:param depth:
:param dropout_rate:
:param n_segmentation_levels:
:param n_labels:
:param optimizer:
:param initial_learning_rate:
:param loss_function:
:param activation_name:
:return:
"""
inputs = Input(input_shape)
current_layer = inputs
level_output_layers = list()
level_filters = list()
for level_number in range(depth):
n_level_filters = (2**level_number) * n_base_filters
level_filters.append(n_level_filters)
if current_layer is inputs:
in_conv = create_convolution_block(current_layer, n_level_filters)
else:
in_conv = create_convolution_block(current_layer,
n_level_filters,
strides=(2, 2, 2))
context_output_layer = create_context_module(in_conv,
n_level_filters,
dropout_rate=dropout_rate)
summation_layer = Add()([in_conv, context_output_layer])
level_output_layers.append(summation_layer)
current_layer = summation_layer
segmentation_layers = list()
for level_number in range(depth - 2, -1, -1):
up_sampling = create_up_sampling_module(current_layer,
level_filters[level_number])
concatenation_layer = concatenate(
[level_output_layers[level_number], up_sampling], axis=1)
localization_output = create_localization_module(
concatenation_layer, level_filters[level_number])
current_layer = localization_output
if level_number < n_segmentation_levels:
segmentation_layers.insert(
0,
Conv3D(n_labels, (1, 1, 1))(current_layer))
output_layer = None
for level_number in reversed(range(n_segmentation_levels)):
segmentation_layer = segmentation_layers[level_number]
if output_layer is None:
output_layer = segmentation_layer
else:
output_layer = Add()([output_layer, segmentation_layer])
if level_number > 0:
output_layer = UpSampling3D(size=(2, 2, 2))(output_layer)
activation_block = Activation(activation_name)(output_layer)
model = Model(inputs=inputs, outputs=activation_block)
model.compile(optimizer=optimizer(lr=initial_learning_rate),
loss=loss_function)
return model
def get_model():
"Returns the model for voxel classification, inputs the whole image + padding"
# Comments track the shape given chunk_size=8 and input_slice_size=256
input = Input(shape=(chunk_size, input_slice_size, input_slice_size,
1)) # 8x256x256
x1 = Conv3D(32, (3, 3, 3),
strides=1,
padding="same",
activation="relu",
kernel_initializer="he_uniform")(input)
x1 = Conv3D(64, (3, 3, 3),
strides=2,
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x1) # 4x128x128x64
x2 = Conv3D(64, (3, 3, 3),
strides=1,
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x1)
x2 = Conv3D(128, (3, 3, 3),
strides=2,
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x2) # 2x64x64x128
x3 = Conv3D(128, (3, 3, 3),
strides=1,
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x2)
x3 = Conv3D(256, (3, 3, 3),
strides=2,
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x3) # 1x32x32x256
x1 = Conv3D(128, (1, 3, 3),
strides=(1, 2, 2),
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x1) # 4x64x64x128
x1 = Conv3D(256, (1, 3, 3),
strides=(1, 2, 2),
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x1) # 4x32x32x256
x2 = Conv3D(256, (1, 3, 3),
strides=(1, 2, 2),
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x2) # 2x32x32x256
x = UpSampling3D((2, 1, 1))(x3) # 2x32x32x256
x = Add()([x, x2])
x = Conv3D(256, (1, 3, 3),
strides=1,
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x)
x = UpSampling3D((2, 1, 1))(x) # 4x32x32x256
x = Add()([x, x1])
x = Conv3D(256, (1, 3, 3),
strides=1,
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x)
x = UpSampling3D((2, 1, 1))(x) # 8x32x32x256
x = Conv3D(256, (1, 3, 3),
strides=1,
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x)
# Number of final_blocks depends on input_slice_size. The output should have a width and height of 1
x = Conv3D(512, (1, 3, 3),
strides=(1, 2, 2),
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x) # 8x16x16
x = Conv3D(512, (1, 3, 3),
strides=(1, 2, 2),
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x) # 8x8x8
x = Conv3D(512, (1, 3, 3),
strides=(1, 2, 2),
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x) # 8x4x4
x = Conv3D(512, (1, 3, 3),
strides=(1, 2, 2),
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x) # 8x2x2
x = Conv3D(512, (1, 3, 3),
strides=(1, 2, 2),
padding="same",
activation="relu",
kernel_initializer="he_uniform")(x) # 8x1x1
assert x.shape[1:4] == (
chunk_size, 1, 1
), "x should be of shape (?, %s, 1, 1, nb_channels), instead it is %s. You can use final_blocks to reduce width and height" % (
chunk_size, x.shape)
predictions = Conv3D(6, (1, 1, 1),
padding="same",
activation="sigmoid",
kernel_initializer="he_uniform")(x)
predictions = Reshape(target_shape=(chunk_size, 6))(predictions)
return Model(inputs=input, outputs=predictions)
strides=(1, 1, 1),
padding='valid',
activation='relu')(conv_3)
act_2 = LeakyReLU(alpha=0.2)(conv_4)
pool_2 = MaxPooling3D(pool_size=(2, 2, 2), strides=None,
padding='valid')(act_2)
conv_5 = Conv3D(64, (3, 3, 3),
strides=(1, 1, 1),
padding='valid',
activation='relu')(pool_2)
conv_6 = Conv3D(64, (3, 3, 3),
strides=(1, 1, 1),
padding='valid',
activation='relu')(conv_5)
act_3 = LeakyReLU(alpha=0.2)(conv_6)
up_1 = UpSampling3D(size=(2, 2, 2))(act_3)
conv_7 = Conv3D(32, (3, 3, 3),
strides=(1, 1, 1),
padding='valid',
activation='relu')(up_1)
conv_8 = Conv3D(32, (3, 3, 3),
strides=(1, 1, 1),
padding='valid',
activation='relu')(conv_7)
act_4 = LeakyReLU(alpha=0.2)(conv_8)
up_2 = UpSampling3D(size=(2, 2, 2))(act_4)
conv_9 = Conv3D(32, (3, 3, 3),
strides=(1, 1, 1),
padding='valid',
activation='relu')(up_2)
conv_10 = Conv3D(32, (3, 3, 3),
def auto_classifier_model(img_shape,
encoding_dim=128,
NUM_CHANNELS=1,
num_of_class=2):
input_shape = (None, img_shape[0], img_shape[1], img_shape[2],
NUM_CHANNELS)
mask_shape = (None, num_of_class)
# use relu activation for hidden layer to guarantee non-negative outputs are passed to the max pooling layer. In such case, as long as the output layer is linear activation, the network can still accomodate negative image intendities, just matter of shift back using the bias term
input_img = Input(shape=input_shape[1:])
mask = Input(shape=mask_shape[1:])
x = input_img
x = conv_block(x, 32, 3, 3, 3)
x = MaxPooling3D((2, 2, 2), padding='same')(x)
x = conv_block(x, 32, 3, 3, 3)
x = MaxPooling3D((2, 2, 2), padding='same')(x)
encoder_conv_shape = [
_.value for _ in x.get_shape()
] # x.get_shape() returns a list of tensorflow.python.framework.tensor_shape.Dimension objects
x = Flatten()(x)
encoded = Dense(encoding_dim,
activation='relu',
activity_regularizer=regularizers.l1(10e-5))(x)
encoder = Model(inputs=input_img, outputs=encoded)
x = BatchNormalization()(x)
x = Dense(encoding_dim,
activation='relu',
activity_regularizer=regularizers.l1(10e-5))(x)
x = Dense(128, activation='relu')(x)
x = Dense(num_of_class, activation='softmax')(x)
prob = x
# classifier output
classifier = Model(inputs=input_img, outputs=prob)
input_img_decoder = Input(shape=encoder.output_shape[1:])
x = input_img_decoder
x = Dense(np.prod(encoder_conv_shape[1:]), activation='relu')(x)
x = Reshape(encoder_conv_shape[1:])(x)
x = UpSampling3D((2, 2, 2))(x)
x = conv_block(x, 32, 3, 3, 3)
x = UpSampling3D((2, 2, 2))(x)
x = conv_block(x, 32, 3, 3, 3)
x = Convolution3D(1, (3, 3, 3), activation='linear', padding='same')(x)
decoded = x
# autoencoder output
decoder = Model(inputs=input_img_decoder, outputs=decoded)
autoencoder = Sequential()
for l in encoder.layers:
autoencoder.add(l)
last = None
for l in decoder.layers:
last = l
autoencoder.add(l)
decoded = autoencoder(input_img)
auto_classifier = Model(inputs=input_img, outputs=[decoded, prob])
auto_classifier.summary()
return auto_classifier
def get_unet(self):
inputs = Input((self.img_rows, self.img_cols, 176, 1))
conv1 = Conv3D(32,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(inputs)
#print ("conv1 shape:",conv1.shape)
conv1 = Conv3D(32,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv1)
#print "conv1 shape:",conv1.shape
pool1 = MaxPooling3D(pool_size=(2, 2, 2))(conv1)
#print "pool1 shape:",pool1.shape
conv2 = Conv3D(64,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool1)
#print "conv2 shape:",conv2.shape
conv2 = Conv3D(64,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv2)
#print "conv2 shape:",conv2.shape
pool2 = MaxPooling3D(pool_size=(2, 2, 2))(conv2)
#print "pool2 shape:",pool2.shape
conv3 = Conv3D(128,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool2)
#print "conv3 shape:",conv3.shape
conv3 = Conv3D(128,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv3)
#print "conv3 shape:",conv3.shape
pool3 = MaxPooling3D(pool_size=(2, 2, 2))(conv3)
#print "pool3 shape:",pool3.shape
conv4 = Conv3D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool3)
conv4 = Conv3D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv4)
drop4 = Dropout(0.5)(conv4)
pool4 = MaxPooling3D(pool_size=(2, 2, 2))(drop4)
conv5 = Conv3D(512,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool4)
conv5 = Conv3D(512,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv5)
drop5 = Dropout(0.5)(conv5)
up6 = Conv3D(256,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(
UpSampling3D(size=(2, 2, 2))(drop5))
merge6 = merge([drop4, up6], mode='concat', concat_axis=4)
conv6 = Conv3D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge6)
conv6 = Conv3D(256,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv6)
up7 = Conv3D(128,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(
UpSampling3D(size=(2, 2, 2))(conv6))
merge7 = merge([conv3, up7], mode='concat', concat_axis=4)
conv7 = Conv3D(128,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge7)
conv7 = Conv3D(128,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv7)
up8 = Conv3D(64,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(
UpSampling3D(size=(2, 2, 2))(conv7))
merge8 = merge([conv2, up8], mode='concat', concat_axis=4)
conv8 = Conv3D(64,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge8)
conv8 = Conv3D(64,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv8)
up9 = Conv3D(32,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(
UpSampling3D(size=(2, 2, 2))(conv8))
merge9 = merge([conv1, up9], mode='concat', concat_axis=4)
conv9 = Conv3D(32,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge9)
conv9 = Conv3D(32,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv9)
conv9 = Conv3D(2,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv9)
conv10 = Conv3D(1, 1, activation='sigmoid')(conv9)
model = Model(input=inputs, output=conv10)
model.compile(optimizer=Adam(lr=1e-4),
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def create_model(input_shape=(4, 160, 192, 160),
n_base_filters=12,
depth=5,
dropout_rate=0.3,
n_segmentation_levels=3,
n_labels=3,
num_outputs=3,
optimizer='adam',
learning_rate=1e-3,
activation_name="sigmoid"):
"""
This function builds a model proposed by Isensee et al. for the BRATS 2017 competition:
https://www.cbica.upenn.edu/sbia/Spyridon.Bakas/MICCAI_BraTS/MICCAI_BraTS_2017_proceedings_shortPapers.pdf
This network is highly similar to the model proposed by Kayalibay et al. "CNN-based Segmentation of Medical
Imaging Data", 2017: https://arxiv.org/pdf/1701.03056.pdf
:param input_shape:
:param n_base_filters:
:param depth:
:param dropout_rate:
:param n_segmentation_levels:
:param n_labels:
:param optimizer:
:param initial_learning_rate:
:param loss_function:
:param activation_name:
:return:
"""
if optimizer.lower() == 'adam':
optimizer = Adam(lr=learning_rate)
inputs = Input(input_shape)
current_layer = inputs
level_output_layers = list()
level_filters = list()
for level_number in range(depth):
n_level_filters = (2**level_number) * n_base_filters
level_filters.append(n_level_filters)
if current_layer is inputs:
in_conv = create_convolution_block(current_layer, n_level_filters)
else:
in_conv = create_convolution_block(current_layer, n_level_filters, strides=(2, 2, 2))
context_output_layer = create_context_module(in_conv, n_level_filters, dropout_rate=dropout_rate)
summation_layer = Add()([in_conv, context_output_layer])
level_output_layers.append(summation_layer)
current_layer = summation_layer
segmentation_layers = list()
for level_number in range(depth - 2, -1, -1):
up_sampling = create_up_sampling_module(current_layer, level_filters[level_number])
concatenation_layer = concatenate([level_output_layers[level_number], up_sampling], axis=1)
localization_output = create_localization_module(concatenation_layer, level_filters[level_number])
current_layer = localization_output
if level_number < n_segmentation_levels:
segmentation_layers.insert(0, create_convolution_block(current_layer, n_filters=n_labels, kernel=(1, 1, 1)))
output_layer = None
for level_number in reversed(range(n_segmentation_levels)):
segmentation_layer = segmentation_layers[level_number]
if output_layer is None:
output_layer = segmentation_layer
else:
output_layer = Add()([output_layer, segmentation_layer])
if level_number > 0:
output_layer = UpSampling3D(size=(2, 2, 2))(output_layer)
activation_block = Activation(activation = activation_name, name='activation_block')(output_layer)
# survival_block = Activation("linear")(summation_layer)
# activation_block = Dense(1, activation=activation_name, name='activation_block')(output_layer)
# flatten = Flatten(name='flatten')(summation_layer)
# survival_block = Dense(1, activation='linear', name='survival_block')(flatten)
survival_conv_1 = Conv3D(filters=n_level_filters, kernel_size=(3, 3, 3), padding='same', strides=(1, 1, 1), name='survival_conv_1')(summation_layer)
survival_conv_2 = Conv3D(filters=n_level_filters, kernel_size=(3, 3, 3), padding='same', strides=(1, 1, 1), name='survival_conv_2')(survival_conv_1)
dropout = SpatialDropout3D(rate=dropout_rate, data_format='channels_first', name='dropout')(survival_conv_2)
survival_conv_3 = Conv3D(filters=n_level_filters, kernel_size=(3, 3, 3), padding='same', strides=(1, 1, 1), name='survival_conv_3')(dropout)
survival_GAP = GlobalAveragePooling3D(name='survival_GAP')(survival_conv_3)
# flatten = Flatten(name='flatten')(survival_GAP)
# survival_block = Activation("linear", name='survival_block')(flatten)
survival_block = Dense(1, activation='linear', name='survival_block')(survival_GAP)
tumortype_conv_1 = Conv3D(filters=n_level_filters, kernel_size=(3, 3, 3), padding='same', strides=(1, 1, 1), name='tumortype_conv_1')(summation_layer)
tumortype_conv_2 = Conv3D(filters=n_level_filters, kernel_size=(3, 3, 3), padding='same', strides=(1, 1, 1), name='tumortype_conv_2')(tumortype_conv_1)
tumortype_dropout = SpatialDropout3D(rate=dropout_rate, data_format='channels_first', name='tumortype_dropout')(tumortype_conv_2)
tumortype_conv_3 = Conv3D(filters=n_level_filters, kernel_size=(3, 3, 3), padding='same', strides=(1, 1, 1), name='tumortype_conv_3')(tumortype_dropout)
tumortype_GAP = GlobalAveragePooling3D(name='tumortype_GAP')(tumortype_conv_3)
# flatten = Flatten(name='flatten')(tumortype_GAP)
# tumortype_block = Activation("linear", name='tumortype_block')(flatten)
tumortype_block = Dense(1, activation='sigmoid', name='tumortype_block')(tumortype_GAP)
model = Model(inputs=inputs, outputs=[activation_block])
# model.compile(optimizer=optimizer(lr=initial_learning_rate), loss=loss_function)
# loss={'activation_block': 'binary_crossentropy', 'survival_block': 'mean_squared_error'}
# assign weights and loss as dictionaries
# functional-api-guide
# loss_weights define the ratio of how much I care about optimizing each one
if num_outputs == 3:
model = Model(inputs=inputs, outputs=[activation_block,tumortype_block,survival_block])
# model.load_weights(weights_dir / f"model_weights_{num_outputs}_outputs.h5", by_name=True)
model.compile(optimizer=optimizer,
loss={'activation_block': weighted_dice_coefficient_loss, 'survival_block': 'mean_squared_error', 'tumortype_block': 'binary_crossentropy'},
loss_weights={'activation_block': 1., 'survival_block': 0.2, 'tumortype_block': 0.2},
metrics={'activation_block': ['accuracy', weighted_dice_coefficient, dice_coefficient], 'survival_block': ['accuracy', 'mae'], 'tumortype_block': ['accuracy']})
if num_outputs == 2:
model = Model(inputs=inputs, outputs=[activation_block,tumortype_block])
# model.load_weights(weights_dir / f"model_weights_{num_outputs}_outputs.h5", by_name=True) # the by_name=True allows you to use a different architecture and bring in the weights from the matching layers
model.compile(optimizer=optimizer,
loss={'activation_block': weighted_dice_coefficient_loss, 'tumortype_block': 'binary_crossentropy'},
loss_weights={'activation_block': 1., 'tumortype_block': 0.2},
metrics={'activation_block': ['accuracy',weighted_dice_coefficient, dice_coefficient], 'tumortype_block': ['accuracy']})
if num_outputs == 1:
model = Model(inputs=inputs, outputs=[activation_block])
# model.load_weights(weights_dir / f"model_weights_{num_outputs}_outputs.h5", by_name=True) # by_name=True allows you to use a different architecture and bring in the weights from the matching layers
model.compile(optimizer=optimizer,
loss={'activation_block': weighted_dice_coefficient_loss},
loss_weights={'activation_block': 1.},
metrics={'activation_block': ['accuracy',weighted_dice_coefficient, dice_coefficient]})
print(model.summary(line_length=150)) # add the parameter that allows me to show everything instead of cutting it off
return model
# model.compile(optimizer=RMSprop(lr=5e-4),
# loss={'activation_block': weighted_dice_coefficient_loss},
# loss_weights={'activation_block': 1.},
# metrics={'activation_block': ['accuracy',weighted_dice_coefficient, dice_coefficient]})
# model.summary(line_length=150) # add the parameter that allows me to show everything instead of cutting it off
# model.save_weights("./weights/model_3_weights.h5")
# print("Saved model to disk")
def unet3_a():
"""
Generate a 3D unet model used in figure 2-S1a
"""
inputs = Input((160, 160, 16, 1))
conv_2 = Conv3D(8, 3, padding='same')(inputs)
conv_2 = LeakyReLU()(conv_2)
conv_2 = BatchNormalization()(conv_2)
conv_2b = Conv3D(16, 3, padding='same')(conv_2)
conv_2b = LeakyReLU()(conv_2b)
conv_2b = BatchNormalization()(conv_2b)
pool_2 = MaxPooling3D(pool_size=(2, 2, 1))(conv_2b)
conv_1 = Conv3D(16, 3, padding='same')(pool_2)
conv_1 = LeakyReLU()(conv_1)
conv_1 = BatchNormalization()(conv_1)
conv_1b = Conv3D(32, 3, padding='same')(conv_1)
conv_1b = LeakyReLU()(conv_1b)
conv_1b = BatchNormalization()(conv_1b)
pool_1 = MaxPooling3D(pool_size=(2, 2, 1))(conv_1b)
conv_0 = Conv3D(32, 3, padding='same')(pool_1)
conv_0 = LeakyReLU()(conv_0)
conv_0 = BatchNormalization()(conv_0)
conv_0b = Conv3D(64, 3, padding='same')(conv_0)
conv_0b = LeakyReLU()(conv_0b)
conv_0b = BatchNormalization()(conv_0b)
pool_0 = MaxPooling3D(pool_size=(2, 2, 1))(conv_0b)
conv_add = Conv3D(64, 3, padding='same')(pool_0)
conv_add = LeakyReLU()(conv_add)
conv_add = BatchNormalization()(conv_add)
conv_addb = Conv3D(64, 3, padding='same')(conv_add)
conv_addb = LeakyReLU()(conv_addb)
conv_addb = BatchNormalization()(conv_addb)
up0 = concatenate([UpSampling3D(size=(2, 2, 1))(conv_addb), conv_0b])
conv_add2 = Conv3D(32, 3, padding='same')(up0)
conv_add2 = LeakyReLU()(conv_add2)
conv_add2 = BatchNormalization()(conv_add2)
conv_add2b = Conv3D(32, 3, padding='same')(conv_add2)
conv_add2b = LeakyReLU()(conv_add2b)
conv_add2b = BatchNormalization()(conv_add2b)
up1 = concatenate([UpSampling3D(size=(2, 2, 1))(conv_add2b), conv_1b])
conv1 = Conv3D(16, 3, padding='same')(up1)
conv1 = LeakyReLU()(conv1)
conv1 = BatchNormalization()(conv1)
conv1b = Conv3D(16, 3, padding='same')(conv1)
conv1b = LeakyReLU()(conv1b)
conv1b = BatchNormalization()(conv1b)
up2 = concatenate([UpSampling3D(size=(2, 2, 1))(conv1b), conv_2b])
conv2 = Conv3D(8, 3, padding='same')(up2)
conv2 = LeakyReLU()(conv2)
conv2 = BatchNormalization()(conv2)
conv2b = Conv3D(8, 3, padding='same')(conv2)
conv2b = LeakyReLU()(conv2b)
conv2b = BatchNormalization()(conv2b)
predictions = Conv3D(1, 1, padding='same', activation='sigmoid')(conv2b)
g_model = Model(inputs=inputs, outputs=predictions)
return g_model
def attention_isensee2017_model(input_shape=(4, 128, 128, 128),
n_base_filters=16,
depth=5,
dropout_rate=0.3,
n_segmentation_levels=3,
n_labels=4,
activation_name="sigmoid",
visualize=False):
"""
This function builds a model proposed by Isensee et al. for the BRATS 2017 competition:
https://www.cbica.upenn.edu/sbia/Spyridon.Bakas/MICCAI_BraTS/MICCAI_BraTS_2017_proceedings_shortPapers.pdf
This network is highly similar to the model proposed by Kayalibay et al. "CNN-based Segmentation of Medical
Imaging Data", 2017: https://arxiv.org/pdf/1701.03056.pdf
:param input_shape:
:param n_base_filters:
:param depth:
:param dropout_rate:
:param n_segmentation_levels:
:param n_labels:
:param optimizer:
:param initial_learning_rate:
:param loss_function:
:param activation_name:
:return:
"""
if visualize:
before_masked = list()
gatting_signal = list()
masks = list()
after_masked = list()
normed_after_masked = list()
inputs = Input(input_shape)
current_layer = inputs
level_output_layers = list()
level_filters = list()
for level_number in range(depth):
n_level_filters = (2**level_number) * n_base_filters
level_filters.append(n_level_filters)
if current_layer is inputs:
in_conv = create_convolution_block(current_layer, n_level_filters)
else:
in_conv = create_convolution_block(current_layer,
n_level_filters,
strides=(2, 2, 2))
context_output_layer = create_context_module(in_conv,
n_level_filters,
dropout_rate=dropout_rate)
summation_layer = Add()([in_conv, context_output_layer])
level_output_layers.append(summation_layer)
current_layer = summation_layer
segmentation_layers = list()
gatting = create_gatting_signal(current_layer, level_filters[-2])
if visualize:
before_masked = level_output_layers[:-1].copy()
before_masked.reverse()
gatting_signal.append(gatting)
for level_number in range(depth - 2, -1, -1):
if level_number > 0:
up_size = 3 * (2**(depth - 1 - level_number), )
masked_current_layer, mask, masked_x = create_attention_gate(
level_output_layers[level_number], gatting,
level_filters[level_number], up_size)
if visualize:
normed_after_masked.append(masked_current_layer)
masks.append(mask)
after_masked.append(masked_x)
else:
masked_current_layer = level_output_layers[level_number]
up_sampling = create_up_sampling_module(current_layer,
level_filters[level_number])
concatenation_layer = concatenate([masked_current_layer, up_sampling],
axis=1)
localization_output = create_localization_module(
concatenation_layer, level_filters[level_number])
current_layer = localization_output
if level_number < n_segmentation_levels:
segmentation_layers.insert(
0,
Conv3D(n_labels, (1, 1, 1))(current_layer))
output_layer = None
for level_number in reversed(range(n_segmentation_levels)):
segmentation_layer = segmentation_layers[level_number]
if output_layer is None:
output_layer = segmentation_layer
else:
output_layer = Add()([output_layer, segmentation_layer])
if level_number > 0:
output_layer = UpSampling3D(size=(2, 2, 2))(output_layer)
activation_block = None
if activation_name == 'sigmoid':
activation_block = Activation(activation_name)(output_layer)
elif activation_name == 'softmax':
activation_block = Softmax(axis=1)(output_layer)
if visualize:
outputs = [activation_block]
outputs += before_masked + gatting_signal + masks + after_masked + normed_after_masked
else:
outputs = activation_block
model = Model(inputs=inputs, outputs=outputs)
return model
def encoder_simple_conv(img_shape, encoding_dim=32, NUM_CHANNELS=1):
# workaround for Dropout to work
# import tensorflow as tf
# tf.python.control_flow_ops = tf
# from seg_util import conv_block
from aitom.classify.deep.unsupervised.autoencoder.seg_util import conv_block
from keras.layers import Input, Dense, Convolution3D, MaxPooling3D, UpSampling3D, Reshape, Flatten
from keras.models import Sequential, Model
from keras import regularizers
input_shape = (None, img_shape[0], img_shape[1], img_shape[2],
NUM_CHANNELS)
# use relu activation for hidden layer to guarantee non-negative outputs
# are passed to the max pooling layer. In such case, as long as the output
# layer is linear activation, the network can still accomodate negative image
# intendities, just matter of shift back using the bias term
input_img = Input(shape=input_shape[1:])
x = input_img
x = conv_block(x, 32, 3, 3, 3)
x = MaxPooling3D((2, 2, 2), border_mode='same')(x)
x = conv_block(x, 32, 3, 3, 3)
x = MaxPooling3D((2, 2, 2), border_mode='same')(x)
# x.get_shape() returns a list of tensorflow.python.framework.tensor_shape.Dimension objects
encoder_conv_shape = [_.value for _ in x.get_shape()]
x = Flatten()(x)
if False:
x = Dense(encoding_dim, activation='relu')(x)
else:
# with sparsity
x = Dense(encoding_dim,
activation='relu',
activity_regularizer=regularizers.l1(10e-5))(x)
encoded = x
encoder = Model(input=input_img, output=encoded)
print('encoder', 'input shape', encoder.input_shape, 'output shape',
encoder.output_shape)
input_img_decoder = Input(shape=encoder.output_shape[1:])
x = input_img_decoder
x = Dense(N.prod(encoder_conv_shape[1:]), activation='relu')(x)
x = Reshape(encoder_conv_shape[1:])(x)
x = UpSampling3D((2, 2, 2))(x)
x = conv_block(x, 32, 3, 3, 3)
x = UpSampling3D((2, 2, 2))(x)
x = conv_block(x, 32, 3, 3, 3)
# keep the output layer linear activation, so that the image intensity can be negative
x = Convolution3D(1, 3, 3, 3, activation='linear', border_mode='same')(x)
decoded = x
decoder = Model(input=input_img_decoder, output=decoded)
autoencoder = Sequential()
if True:
# model with expanded layers, and hopefully allow parallel training
for l in encoder.layers:
autoencoder.add(l)
for l in decoder.layers:
autoencoder.add(l)
else:
# build encoder according to ~/src/tmp/proj/gan/dcgan.py
autoencoder.add(encoder)
autoencoder.add(decoder)
print('autoencoder layers:')
for l in autoencoder.layers:
print(l.output_shape)
return {'autoencoder': autoencoder, 'encoder': encoder, 'decoder': decoder}
def unet(vol_size, enc_nf, dec_nf, full_size=True):
"""
unet network for voxelmorph
Args:
vol_size: volume size. e.g. (256, 256, 256)
enc_nf: encoder filters. right now it needs to be to 1x4.
e.g. [16,32,32,32]
TODO: make this flexible.
dec_nf: encoder filters. right now it's forced to be 1x7.
e.g. [32,32,32,32,8,8,3]
TODO: make this flexible.
full_size
"""
# inputs
src = Input(shape=(vol_size[0], vol_size[1], vol_size[2], 1))
tgt = Input(shape=(vol_size[0], vol_size[1], vol_size[2], 1))
x_in = concatenate([src, tgt], 4)
#x_in = ZeroPadding3D(((3,2), (2,1), (3,2)))(x_in)
print(K.int_shape(x_in))
# down-sample path.
x0 = myConv(x_in, enc_nf[0], 2)
x1 = myConv(x0, enc_nf[1], 2)
x2 = myConv(x1, enc_nf[2], 2)
x3 = myConv(x2, enc_nf[3], 2)
# up-sample path.
x = myConv(x3, dec_nf[0])
x = UpSampling3D()(x)
x2 = ZeroPadding3D(((0, 0), (0, 1), (0, 0)))(x2)
x = concatenate([x, x2])
x = myConv(x, dec_nf[1])
x = UpSampling3D()(x)
x1 = ZeroPadding3D(((0, 0), (1, 2), (0, 0)))(x1)
x = concatenate([x, x1])
x = myConv(x, dec_nf[2])
x = UpSampling3D()(x)
x0 = ZeroPadding3D(((0, 0), (3, 3), (0, 0)))(x0)
x = concatenate([x, x0])
x = myConv(x, dec_nf[3])
x = myConv(x, dec_nf[4])
if full_size:
x = UpSampling3D()(x)
x_in = ZeroPadding3D(((0, 0), (6, 6), (0, 0)))(x_in)
x = concatenate([x, x_in])
x = myConv(x, dec_nf[5])
# optional convolution
if (len(dec_nf) == 8):
x = myConv(x, dec_nf[6])
# transform the results into a flow.
flow = Conv3D(dec_nf[-1],
kernel_size=3,
padding='same',
kernel_initializer=RandomNormal(mean=0.0, stddev=1e-5),
name='flow')(x)
flow = Lambda(lambda x: x[:, :, 6:-6, :, :])(flow)
# warp the source with the flow
y = Dense3DSpatialTransformer()([src, flow])
# prepare model
model = Model(inputs=[src, tgt], outputs=[y, flow])
# warped_img, forward_flow = model([src, tgt])
# warped_back, backward_flow = model([warped_img, src])
# insert the refinement model
flow_in = Input(shape=(vol_size[0], vol_size[1], vol_size[2], 3))
flow_out = Refine_Block(flow_in, 16, strides=1)
model_refine = Model(inputs=flow_in, outputs=flow_out)
# build the full model
y_ini, flow_ini = model([src, tgt])
flow_refined = model_refine(flow_ini)
y_refined = Dense3DSpatialTransformer()([src, flow_refined])
full_model = Model([src, tgt], outputs=[y_refined, flow_refined])
return full_model, y_ini, flow_ini
def test_delete_channels_upsampling3d(channel_index, data_format):
layer = UpSampling3D([2, 3, 2], data_format=data_format)
layer_test_helper_flatten_3d(layer, channel_index, data_format=data_format)
def buildModel(inputShape, args):
x,y,z = inputShape
input_img = Input(shape=(x, y, z, 1))
x = input_img
if args.batchNorm: x = BatchNormalization()(x)
fo = args.filters
fi = fo*2
b = args.bconv
x = Convolution3D(fo, (b, b, b), activation='relu', padding='same')(x)
x = Convolution3D(fo, (3, 3, 3), activation='relu', padding='same')(x)
#x = MaxPooling3D((2, 2, 2), border_mode=border)(x)
x = MaxPooling3D(name='pool1')(x)
#x = Convolution3D(8, cs, cs, cs, activation='relu', border_mode=border)(x)
#x = MaxPooling3D((2, 2, 2 ), border_mode=border)(x)
x = Convolution3D(fi, (3, 3, 3), activation='relu', padding='same')(x)
if args.doubleLayers: x = Convolution3D(fi, (3, 3, 3), activation='relu', padding='same')(x)
x = MaxPooling3D(name='pool2')(x)
#x = Convolution3D(8, cs, cs, cs, activation='relu', border_mode=border)(x)
#x = Convolution3D(8, cs, cs, cs, activation='relu', border_mode=border)(x)
#x = MaxPooling3D((2, 2, 2), border_mode=border)(x)
encoded = x
encoder = Model(inputs=[input_img], outputs=[encoded])
#flat = Flatten()(encoded)
flat = GlobalAveragePooling3D()(encoded)
if args.dropout: flat = Dropout(args.dropout)(flat)
shared = Dense(args.sharedNeurons)(flat)
nodule = Dense(1, activation='sigmoid', name='nodule')(shared)
loss = {'nodule': 'binary_crossentropy'}
metrics = {'nodule': 'accuracy'}
db = Dense(16)(flat)
if args.diam == 'linear':
diam = Dense(1, activation='relu', name='diam')(db)
loss['diam'] = 'mse'
metrics['diam'] = 'mae'
elif args.diam == 'softmax':
diam = Dense(3, activation='softmax', name='diam')(db)
loss['diam'] = 'categorical_crossentropy'
metrics['diam'] = 'accuracy'
if args.autoencoder:
loss['imgOut'] = 'mse'
metrics['imgOut'] = 'mae'
x = UpSampling3D()(x)
x = Convolution3D(fi, (3, 3, 3), activation='relu', padding='same')(x)
if args.doubleLayers: x = Convolution3D(fi, (3, 3, 3), activation='relu', padding='same')(x)
x = UpSampling3D()(x)
if args.doubleLayers: x = Convolution3D(fo, (3, 3, 3), activation='relu', padding='same')(x)
decoded = Convolution3D(1, (3, 3, 3), activation='relu', padding='same', name='imgOut')(x)
model = Model(inputs=[input_img], outputs=[nodule, diam, decoded], name='multiOut')
else:
model = Model(inputs=[input_img], outputs=[nodule, diam], name='multiOut')
print 'hidden layer shape: ', encoder.output_shape
#model.compile( optimizer='adadelta', loss=loss, metrics=metrics )
model.compile( optimizer=args.optimizer, loss=loss, metrics=metrics )
print model.summary()
return model
def unet_model_xd3_2_6l_grid(
nb_filter=48,
dim=5,
clen=3,
img_rows=224,
img_cols=224
): # NOTE that this procedure is/should be used with img_rows & img_cols as None
# aiming for architecture similar to the http://cs231n.stanford.edu/reports2016/317_Report.pdf
# Our model is six layers deep, consisting of a series of three CONV-RELU-POOL layyers (with 32, 32, and 64 3x3 filters), a CONV-RELU layer (with 128 3x3 filters), three UPSCALE-CONV-RELU lay- ers (with 64, 32, and 32 3x3 filters), and a final 1x1 CONV- SIGMOID layer to output pixel-level predictions. Its struc- ture resembles Figure 2, though with the number of pixels, filters, and levels as described here
## 3D CNN version of a previously developed unet_model_xd_6j
zconv = clen
inputs = Input((1, dim, img_rows, img_cols))
conv1 = Convolution3D(nb_filter,
zconv,
clen,
clen,
activation='relu',
border_mode='same')(inputs)
conv1 = Convolution3D(nb_filter,
zconv,
clen,
clen,
activation='relu',
border_mode='same')(conv1)
pool1 = MaxPooling3D(pool_size=(2, 2, 2))(conv1)
conv2 = Convolution3D(2 * nb_filter,
zconv,
clen,
clen,
activation='relu',
border_mode='same')(pool1)
conv2 = Convolution3D(2 * nb_filter,
zconv,
clen,
clen,
activation='relu',
border_mode='same')(conv2)
pool2 = MaxPooling3D(pool_size=(2, 2, 2))(conv2)
conv4 = Convolution3D(4 * nb_filter,
zconv,
clen,
clen,
activation='relu',
border_mode='same')(pool2)
conv4 = Convolution3D(4 * nb_filter,
zconv,
clen,
clen,
activation='relu',
border_mode='same')(conv4)
up6 = merge([UpSampling3D(size=(2, 2, 2))(conv4), conv2],
mode='concat',
concat_axis=1)
conv6 = Convolution3D(2 * nb_filter,
zconv,
clen,
clen,
activation='relu',
border_mode='same')(up6)
conv6 = Convolution3D(2 * nb_filter,
zconv,
clen,
clen,
activation='relu',
border_mode='same')(conv6)
up7 = merge([UpSampling3D(size=(2, 2, 2))(conv6), conv1],
mode='concat',
concat_axis=1) # original - only works for even dim
conv7 = Convolution3D(nb_filter,
zconv,
clen,
clen,
activation='relu',
border_mode='same')(up7)
conv7 = Convolution3D(nb_filter,
zconv,
clen,
clen,
activation='relu',
border_mode='same')(conv7)
pool11 = MaxPooling3D(pool_size=(2, 1, 1))(conv7)
conv12 = Convolution3D(2 * nb_filter,
zconv,
1,
1,
activation='relu',
border_mode='same')(pool11)
conv12 = Convolution3D(2 * nb_filter,
zconv,
1,
1,
activation='relu',
border_mode='same')(conv12)
pool12 = MaxPooling3D(pool_size=(2, 1, 1))(conv12)
conv13 = Convolution3D(2 * nb_filter,
zconv,
1,
1,
activation='relu',
border_mode='same')(pool12)
conv13 = Convolution3D(2 * nb_filter,
zconv,
1,
1,
activation='relu',
border_mode='same')(conv13)
pool13 = MaxPooling3D(pool_size=(2, 1, 1))(conv13)
if (dim < 16):
conv8 = Convolution3D(1, 1, 1, 1, activation='sigmoid')(pool13)
else: # need one extra layer to get to 1D x 2D mask ...
conv14 = Convolution3D(2 * nb_filter,
zconv,
1,
1,
activation='relu',
border_mode='same')(pool13)
conv14 = Convolution3D(2 * nb_filter,
zconv,
1,
1,
activation='relu',
border_mode='same')(conv14)
pool14 = MaxPooling3D(pool_size=(2, 1, 1))(conv14)
conv8 = Convolution3D(1, 1, 1, 1, activation='sigmoid')(pool14)
model = Model(input=inputs, output=conv8)
model.compile(optimizer=Adam(lr=1e-4),
loss=dice_coef_loss,
metrics=[dice_coef])
#model.compile(optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0), loss=dice_coef_loss, metrics=[dice_coef])
return model
def createModel(patchSize, numClasses, usingClassification=False):
if K.image_data_format() == 'channels_last':
bn_axis = -1
else:
bn_axis = 1
input_tensor = Input(shape=(patchSize[0], patchSize[1], patchSize[2], 1))
# first stage
x = Conv3D(filters=16,
kernel_size=(5, 5, 5),
strides=(1, 1, 1),
padding='same',
kernel_initializer='he_normal')(input_tensor)
x = BatchNormalization(axis=bn_axis)(x)
x = LeakyReLU(alpha=0.01)(x)
x_after_stage_1 = Add()([input_tensor, x])
# first down convolution
x_down_conv_1 = projection_block_3D(x_after_stage_1,
filters=(32, 32),
kernel_size=(2, 2, 2),
stage=1,
block=1,
se_enabled=False,
se_ratio=16)
# second stage
x = identity_block_3D(x_down_conv_1,
filters=(32, 32),
kernel_size=(3, 3, 3),
stage=2,
block=1,
se_enabled=False,
se_ratio=16)
x_after_stage_2 = identity_block_3D(x,
filters=(32, 32),
kernel_size=(3, 3, 3),
stage=2,
block=2,
se_enabled=False,
se_ratio=16)
# second down convolution
x_down_conv_2 = projection_block_3D(x_after_stage_2,
filters=(64, 64),
kernel_size=(2, 2, 2),
stage=2,
block=3,
se_enabled=False,
se_ratio=16)
# third stage
x = identity_block_3D(x_down_conv_2,
filters=(64, 64),
kernel_size=(3, 3, 3),
stage=3,
block=1,
se_enabled=False,
se_ratio=16)
x_after_stage_3 = identity_block_3D(x,
filters=(64, 64),
kernel_size=(3, 3, 3),
stage=3,
block=2,
se_enabled=False,
se_ratio=16)
#x = identity_block_3D(x, filters=(64, 64), kernel_size=(3, 3, 3), stage=3, block=3, se_enabled=False, se_ratio=16)
# third down convolution
x_down_conv_3 = projection_block_3D(x_after_stage_3,
filters=(128, 128),
kernel_size=(2, 2, 2),
stage=3,
block=4,
se_enabled=False,
se_ratio=16)
# fourth stage
x = identity_block_3D(x_down_conv_3,
filters=(128, 128),
kernel_size=(3, 3, 3),
stage=4,
block=1,
se_enabled=False,
se_ratio=16)
x_after_stage_4 = identity_block_3D(x,
filters=(128, 128),
kernel_size=(3, 3, 3),
stage=4,
block=2,
se_enabled=False,
se_ratio=16)
#x = identity_block_3D(x, filters=(128, 128), kernel_size=(3, 3, 3), stage=4, block=3, se_enabled=False, se_ratio=16)
### end of encoder path
if usingClassification:
# use x_after_stage_4 as quantification output
# global average pooling
x_class = GlobalAveragePooling3D(
data_format='channels_last')(x_after_stage_4)
# fully-connected layer
classification_output = Dense(units=numClasses,
activation='softmax',
kernel_initializer='he_normal',
name='classification_output')(x_class)
### decoder path
# first 3D upsampling
x = UpSampling3D(size=(2, 2, 2),
data_format=K.image_data_format())(x_after_stage_4)
x = Conv3D(filters=64,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
kernel_initializer='he_normal')(x)
x = BatchNormalization(axis=bn_axis)(x)
x = LeakyReLU(alpha=0.01)(x)
#x = concatenate([x, x_after_stage_3], axis=bn_axis)
x = Add()([x_after_stage_3, x])
# first decoder stage
x = identity_block_3D(x,
filters=(64, 64),
kernel_size=(3, 3, 3),
stage=6,
block=1,
se_enabled=False,
se_ratio=16)
#x = identity_block_3D(x, filters=(128, 128), kernel_size=(3, 3, 3), stage=6, block=2, se_enabled=False, se_ratio=16)
# second 3D upsampling
x = UpSampling3D(size=(2, 2, 2), data_format=K.image_data_format())(x)
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
kernel_initializer='he_normal')(x)
x = BatchNormalization(axis=bn_axis)(x)
x = LeakyReLU(alpha=0.01)(x)
#x = concatenate([x, x_after_stage_2], axis=bn_axis)
x = Add()([x_after_stage_2, x])
# second decoder stage
x = identity_block_3D(x,
filters=(32, 32),
kernel_size=(3, 3, 3),
stage=7,
block=1,
se_enabled=False,
se_ratio=16)
#x = identity_block_3D(x, filters=(64, 64), kernel_size=(3, 3, 3), stage=7, block=2, se_enabled=False, se_ratio=16)
# third 3D upsampling
x = UpSampling3D(size=(2, 2, 2), data_format=K.image_data_format())(x)
x = Conv3D(filters=16,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
kernel_initializer='he_normal')(x)
x = BatchNormalization(axis=bn_axis)(x)
x = LeakyReLU(alpha=0.01)(x)
#x = concatenate([x, x_after_stage_1], axis=bn_axis)
x = Add()([x_after_stage_1, x])
# third decoder stage
x = identity_block_3D(x,
filters=(16, 16),
kernel_size=(3, 3, 3),
stage=9,
block=1,
se_enabled=False,
se_ratio=16)
### End of decoder
### last segmentation segments
# 1x1x1-Conv3 produces 2 featuremaps for probabilistic segmentations of the foreground and background
x = Conv3D(filters=2,
kernel_size=(1, 1, 1),
strides=(1, 1, 1),
padding='same',
kernel_initializer='he_normal',
name='conv_veryEnd')(x)
x = BatchNormalization(axis=bn_axis)(x)
x = LeakyReLU(alpha=0.01)(x)
segmentation_output = Softmax(axis=bn_axis, name='segmentation_output')(x)
# create model
if usingClassification:
cnn = Model(inputs=[input_tensor],
outputs=[segmentation_output, classification_output],
name='3D-VResFCN-Classification')
sModelName = cnn.name
else:
cnn = Model(inputs=[input_tensor],
outputs=[segmentation_output],
name='3D-VResFCN')
sModelName = cnn.name
return cnn, sModelName
def create_up_sampling_module(input_layer, n_filters, size=(2, 2, 2)):
up_sample = UpSampling3D(size=size)(input_layer)
convolution = create_convolution_block(up_sample, n_filters)
return convolution
def build_model(input_shape=(4, 160, 192, 128),
output_channels=3,
weight_L2=0.1,
weight_KL=0.1):
"""
build_model(input_shape=(4, 160, 192, 128), output_channels=3, weight_L2=0.1, weight_KL=0.1)
-------------------------------------------
Creates the model used in the BRATS2018 winning solution
by Myronenko A. (https://arxiv.org/pdf/1810.11654.pdf)
Parameters
----------
`input_shape`: A 4-tuple, optional.
Shape of the input image. Must be a 4D image of shape (c, H, W, D),
where, each of H, W and D are divisible by 2^4, and c is divisible by 4.
Defaults to the crop size used in the paper, i.e., (4, 160, 192, 128).
`output_channels`: An integer, optional.
The no. of channels in the output. Defaults to 3 (BraTS 2018 format).
`weight_L2`: A real number, optional
The weight to be given to the L2 loss term in the loss function. Adjust to get best
results for your task. Defaults to 0.1.
`weight_KL`: A real number, optional
The weight to be given to the KL loss term in the loss function. Adjust to get best
results for your task. Defaults to 0.1.
Returns
-------
`model`: A keras.models.Model instance
The created model.
"""
c, H, W, D = input_shape
assert len(input_shape) == 4, "Input shape must be a 4-tuple"
assert (c % 4) == 0, "The no. of channels must be divisible by 4"
assert (H % 16) == 0 and (W % 16) == 0 and (D % 16) == 0, \
"All the input dimensions must be divisible by 16"
# -------------------------------------------------------------------------
# Encoder
# -------------------------------------------------------------------------
## Input Layer
inp = Input(input_shape)
## The Initial Block
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_x1')(inp)
## Dropout (0.2)
x = Dropout(0.2)(x)
## Green Block x1 (output filters = 32)
x1 = green_block(x, 32, name='x1')
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_32')(x1)
## Green Block x2 (output filters = 64)
x = green_block(x, 64, name='Enc_64_1')
x2 = green_block(x, 64, name='x2')
x = Conv3D(filters=64,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_64')(x2)
## Green Blocks x2 (output filters = 128)
x = green_block(x, 128, name='Enc_128_1')
x3 = green_block(x, 128, name='x3')
x = Conv3D(filters=128,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_128')(x3)
## Green Blocks x4 (output filters = 256)
x = green_block(x, 256, name='Enc_256_1')
x = green_block(x, 256, name='Enc_256_2')
x = green_block(x, 256, name='Enc_256_3')
x4 = green_block(x, 256, name='x4')
# -------------------------------------------------------------------------
# Decoder
# -------------------------------------------------------------------------
## GT (Groud Truth) Part
# -------------------------------------------------------------------------
### Green Block x1 (output filters=128)
x = Conv3D(filters=128,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_128')(x4)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_128')(x)
x = Add(name='Input_Dec_GT_128')([x, x3])
x = green_block(x, 128, name='Dec_GT_128')
### Green Block x1 (output filters=64)
x = Conv3D(filters=64,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_64')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_64')(x)
x = Add(name='Input_Dec_GT_64')([x, x2])
x = green_block(x, 64, name='Dec_GT_64')
### Green Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_32')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_32')(x)
x = Add(name='Input_Dec_GT_32')([x, x1])
x = green_block(x, 32, name='Dec_GT_32')
### Blue Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_Dec_GT_Output')(x)
### Output Block
out_GT = Conv3D(
filters=output_channels, # No. of tumor classes is 3
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
activation='sigmoid',
name='Dec_GT_Output')(x)
## VAE (Variational Auto Encoder) Part
# -------------------------------------------------------------------------
### VD Block (Reducing dimensionality of the data)
x = GroupNormalization(groups=8, axis=1, name='Dec_VAE_VD_GN')(x4)
x = LeakyReLU(alpha=0.2)(x)
x = Conv3D(filters=16,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Dec_VAE_VD_Conv3D')(x)
# Not mentioned in the paper, but the author used a Flattening layer here.
x = Flatten(name='Dec_VAE_VD_Flatten')(x)
x = Dense(256, name='Dec_VAE_VD_Dense')(x)
### VDraw Block (Sampling)
z_mean = Dense(128, name='Dec_VAE_VDraw_Mean')(x)
z_var = Dense(128, name='Dec_VAE_VDraw_Var')(x)
x = Lambda(sampling, name='Dec_VAE_VDraw_Sampling')([z_mean, z_var])
### VU Block (Upsizing back to a depth of 256)
x = Dense((c // 4) * (H // 16) * (W // 16) * (D // 16))(x)
x = Activation('relu')(x)
x = Reshape(((c // 4), (H // 16), (W // 16), (D // 16)))(x)
x = Conv3D(filters=256,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
activation='relu',
name='Dec_VAE_ReduceDepth_256')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_256')(x)
### Green Block x1 (output filters=128)
x = Conv3D(filters=128,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_128')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_128')(x)
x = green_block(x, 128, name='Dec_VAE_128')
### Green Block x1 (output filters=64)
x = Conv3D(filters=64,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_64')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_64')(x)
x = green_block(x, 64, name='Dec_VAE_64')
### Green Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_32')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_32')(x)
x = green_block(x, 32, name='Dec_VAE_32')
### Blue Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_Dec_VAE_Output')(x)
### Output Block
out_VAE = Conv3D(filters=4,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_Output')(x)
# Build and Compile the model
out = out_GT
model = Model(inp, out) # Create the model
model.compile(adam(lr=1e-4),
loss(input_shape,
inp,
out_VAE,
z_mean,
z_var,
weight_L2=weight_L2,
weight_KL=weight_KL),
metrics=[dice_coefficient])
return model
def build_model(input_shape=(4, 160, 192, 128)):
"""
build_model(input_shape=(4, 160, 192, 128))
-------------------------------------------
Creates the model used in the BRATS2018 winning solution
by Myronenko A. (https://arxiv.org/pdf/1810.11654.pdf)
Parameters
----------
`input_shape`: A 4-tuple, optional.
Shape of the input image. Must be a 4D image of shape (c, H, W, D),
where, each of H, W and D are divisible by 2^4. Defaults to the crop
size used in the paper, i.e., (4, 160, 192, 128).
Returns
-------
`model`: A keras.models.Model instance
The created model.
"""
c, H, W, D = input_shape
assert len(input_shape) == 4, "Input shape must be a 4-tuple"
assert ~(H % 16) and ~(W % 16) and ~(D % 16), \
"All the input dimensions must be divisible by 16"
# -------------------------------------------------------------------------
# Encoder
# -------------------------------------------------------------------------
## Input Layer
inp = Input(input_shape)
## The Initial Block
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_x1')(inp)
## Dropout (0.2)
x = Dropout(0.2)(x)
## Green Block x1 (output filters = 32)
x1 = green_block(x, 32, name='x1')
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_32')(x1)
## Green Block x2 (output filters = 64)
x = green_block(x, 64, name='Enc_64_1')
x2 = green_block(x, 64, name='x2')
x = Conv3D(filters=64,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_64')(x2)
## Green Blocks x2 (output filters = 128)
x = green_block(x, 128, name='Enc_128_1')
x3 = green_block(x, 128, name='x3')
x = Conv3D(filters=128,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_128')(x3)
## Green Blocks x4 (output filters = 256)
x = green_block(x, 256, name='Enc_256_1')
x = green_block(x, 256, name='Enc_256_2')
x = green_block(x, 256, name='Enc_256_3')
x4 = green_block(x, 256, name='x4')
# -------------------------------------------------------------------------
# Decoder
# -------------------------------------------------------------------------
## GT (Groud Truth) Part
# -------------------------------------------------------------------------
### Green Block x1 (output filters=128)
x = Conv3D(filters=128,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_128')(x4)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_128')(x)
x = Add(name='Input_Dec_GT_128')([x, x3])
x = green_block(x, 128, name='Dec_GT_128')
### Green Block x1 (output filters=64)
x = Conv3D(filters=64,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_64')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_64')(x)
x = Add(name='Input_Dec_GT_64')([x, x2])
x = green_block(x, 64, name='Dec_GT_64')
### Green Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_32')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_32')(x)
x = Add(name='Input_Dec_GT_32')([x, x1])
x = green_block(x, 32, name='Dec_GT_32')
### Blue Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_Dec_GT_Output')(x)
### Output Block
out_GT = Conv3D(
filters=3, # No. of tumor classes is 3
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
activation='sigmoid',
name='Dec_GT_Output')(x)
## VAE (Variational Auto Encoder) Part
# -------------------------------------------------------------------------
### VD Block (Reducing dimensionality of the data)
x = GroupNormalization(groups=8, axis=1, name='Dec_VAE_VD_GN')(x4)
x = Activation('relu', name='Dec_VAE_VD_relu')(x)
x = Conv3D(filters=16,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Dec_VAE_VD_Conv3D')(x)
# Not mentioned in the paper, but the author used a Flattening layer here.
x = Flatten(name='Dec_VAE_VD_Flatten')(x)
x = Dense(256, name='Dec_VAE_VD_Dense')(x)
### VDraw Block (Sampling)
z_mean = Dense(128, name='Dec_VAE_VDraw_Mean')(x)
z_var = Dense(128, name='Dec_VAE_VDraw_Var')(x)
x = Lambda(sampling, name='Dec_VAE_VDraw_Sampling')([z_mean, z_var])
### VU Block (Upsizing back to a depth of 256)
x = Dense(1 * 10 * 12 * 8)(x)
x = Activation('relu')(x)
x = Reshape((1, 10, 12, 8))(x)
x = Conv3D(filters=256,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_256')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_256')(x)
### Green Block x1 (output filters=128)
x = Conv3D(filters=128,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_128')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_128')(x)
x = green_block(x, 128, name='Dec_VAE_128')
### Green Block x1 (output filters=64)
x = Conv3D(filters=64,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_64')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_64')(x)
x = green_block(x, 64, name='Dec_VAE_64')
### Green Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_32')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_32')(x)
x = green_block(x, 32, name='Dec_VAE_32')
### Blue Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_Dec_VAE_Output')(x)
### Output Block
out_VAE = Conv3D(filters=4,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_Output')(x)
# Build and Compile the model
out = out_GT
model = Model(inp, out) # Create the model
model.compile(adam(lr=1e-4),
loss(input_shape, inp, out_VAE, z_mean, z_var),
metrics=['accuracy'])
return model
def unet_model(input_image_size,
n_labels=1,
layers=4,
lowest_resolution=16,
convolution_kernel_size=(5, 5, 5),
deconvolution_kernel_size=(5, 5, 5),
pool_size=(2, 2, 2),
strides=(2, 2, 2),
mode='classification',
output_activation='tanh',
init_lr=0.0001):
layers = np.arange(layers)
number_of_classification_labels = n_labels
inputs = Input(shape=input_image_size)
## ENCODING PATH ##
encoding_convolution_layers = []
pool = None
for i in range(len(layers)):
number_of_filters = lowest_resolution * 2**(layers[-i])
print("There are a number of " + str(number_of_filters) + " layers.")
if i == 0:
conv = Conv3D(filters=number_of_filters,
kernel_size=convolution_kernel_size,
activation='relu',
padding='same')(inputs)
print("Conv Layer: " + str(i))
print(conv)
#conv = conv[:,0,:,:,:]
else:
conv = Conv3D(filters=number_of_filters,
kernel_size=convolution_kernel_size,
activation='relu',
padding='same')(pool)
print("Conv Layer: " + str(i))
print(conv)
encoding_convolution_layers.append(
Conv3D(filters=number_of_filters,
kernel_size=convolution_kernel_size,
activation='relu',
padding='same')(conv))
if i < len(layers) - 1:
pool = MaxPooling3D(pool_size=pool_size,
data_format="channels_first")(
encoding_convolution_layers[i])
print("Pool Layer: " + str(i))
print(pool)
## DECODING PATH ##
outputs = encoding_convolution_layers[len(layers) - 1]
for i in range(1, len(layers)):
number_of_filters = lowest_resolution * 2**(len(layers) - layers[i] -
1)
tmp_deconv = Conv3DTranspose(filters=number_of_filters,
kernel_size=deconvolution_kernel_size,
padding='same')(outputs)
print("Deconv Layer: " + str(i))
print(tmp_deconv)
tmp_deconv = UpSampling3D(size=pool_size,
data_format="channels_first")(tmp_deconv)
print("Upsampling Layer: " + str(i))
print(tmp_deconv)
outputs = Concatenate(axis=3)(
[tmp_deconv, encoding_convolution_layers[len(layers) - i - 1]])
outputs = Conv3D(filters=number_of_filters,
kernel_size=convolution_kernel_size,
activation='relu',
padding='same')(outputs)
outputs = Conv3D(filters=number_of_filters,
kernel_size=convolution_kernel_size,
activation='relu',
padding='same')(outputs)
if mode == 'classification':
if number_of_classification_labels == 1:
outputs = Conv3D(filters=number_of_classification_labels,
kernel_size=(1, 1, 1),
activation='sigmoid')(outputs)
else:
outputs = Conv3D(filters=number_of_classification_labels,
kernel_size=(1, 1, 1),
activation='softmax')(outputs)
unet_model = Model(inputs=inputs, outputs=outputs)
if number_of_classification_labels == 1:
unet_model.compile(loss=loss_dice_coefficient_error,
optimizer=opt.Adam(lr=init_lr),
metrics=[dice_coefficient])
else:
unet_model.compile(
loss='categorical_crossentropy',
optimizer=opt.Adam(lr=init_lr),
metrics=['accuracy', 'categorical_crossentropy'])
elif mode == 'regression':
outputs = Conv3D(filters=number_of_classification_labels,
kernel_size=(1, 1),
activation=output_activation)(outputs)
unet_model = Model(inputs=inputs, outputs=outputs)
unet_model.compile(loss='mse', optimizer=opt.Adam(lr=init_lr))
else:
raise ValueError(
'mode must be either `classification` or `regression`')
return unet_model
def __init__(self, input_shape=(48, 48, 48, 1), n_classes=10, n_filters=4):
"""
5D tensor with shape: (samples, channels, conv_dim1, conv_dim2, conv_dim3) if dim_ordering='th' or
5D tensor with shape: (samples, conv_dim1, conv_dim2, conv_dim3, channels) if dim_ordering='tf'.
"""
# input_img = Input(shape=(1, 28, 28, 28)) # th =(nChan, nFrames, xPix, yPix) or (nChan, z, x, y)
input_img = Input(
shape=input_shape
) # th =(nChan, nFrames, xPix, yPix) or (nChan, z, x, y)
img_chans = 1
x = Conv3D(n_filters, 3, 3, 3, activation='relu',
border_mode='same')(input_img)
x = MaxPooling3D((2, 2, 2), border_mode='same')(x)
x = Conv3D(n_filters * 8,
3,
3,
3,
activation='relu',
border_mode='same')(x)
x = MaxPooling3D((2, 2, 2), border_mode='same')(x)
x = Conv3D(n_filters * 16,
3,
3,
3,
activation='relu',
border_mode='same')(x)
self.z = MaxPooling3D((2, 2, 2), border_mode='same')(x)
# at this point the representation is (4, 4, 4, 8) i.e. 512-dimensional
flat = Flatten()(self.z)
classer_base = Dense(n_classes,
init='normal',
activation='softmax',
name='classer_output')(flat)
x = Conv3D(n_filters * 16,
3,
3,
3,
activation='relu',
border_mode='same')(self.z)
x = UpSampling3D((2, 2, 2))(x)
x = Conv3D(n_filters * 8,
3,
3,
3,
activation='relu',
border_mode='same')(x)
x = UpSampling3D((2, 2, 2))(x)
# x = Conv3D(16, 3, 3, 3, activation='relu', border_mode='valid')(x)
x = UpSampling3D((2, 2, 2))(x)
# smash the extra channels down
x = Conv3D(img_chans,
3,
3,
3,
activation='sigmoid',
border_mode='same')(x)
# print(input_shape)
# x = Dense(input_shape)(x) # fudge to fix output shape
self.decoded = x
self.autoencoder = Model(input_img, self.decoded)
self.classifier = Model(input_img, classer_base)
self.classifier.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
def res_vae(model_path, num_channels, dims, ds, learning_rate):
intermediate_dim = 1024
latent_dim = 2
epsilon_std = 1.0
# full vae
input_img = Input(shape=dims)
x = Conv3D(64, (3, 3), strides=(2, 2), activation='relu',
padding='same')(input_img)
x = BatchNormalization()(x)
y = Activation('relu')(x)
x = Conv3D(64, (3, 3), strides=(1, 1), activation='relu',
padding='same')(y)
x = BatchNormalization()(x)
x = add([x, y])
x = Activation('relu')(x)
for _ in range(7):
x = Conv3D(32, (3, 3), strides=(2, 2), padding='same')(x)
x = BatchNormalization()(x)
y = Activation('relu')(x)
x = Conv3D(32, (3, 3), strides=(1, 1), padding='same')(y)
x = BatchNormalization()(x)
x = add([x, y])
x = Activation('relu')(x)
flat = Flatten()(x)
hidden = Dense(intermediate_dim, activation='relu')(flat)
z_mean = Dense(latent_dim)(hidden)
z_log_var = Dense(latent_dim)(hidden)
def sampling(args):
z_mean, z_log_var = args
epsilon = K.random_normal(shape=(K.shape(z_mean)[0], latent_dim),
mean=0.,
stddev=epsilon_std)
return z_mean + K.exp(z_log_var) * epsilon
z = Lambda(sampling)([z_mean, z_log_var])
x = Dense(intermediate_dim, activation='relu')(z)
x = Dense(2 * 2 * 32, activation='relu')(x)
x = Reshape(target_shape=(2, 2, 32))(x)
x = UpSampling3D((2, 2))(x)
for _ in range(7):
x = Conv3D(32, (3, 3), strides=(1, 1), padding='same')(x)
x = BatchNormalization()(x)
y = Activation('relu')(x)
x = Conv3D(32, (3, 3), strides=(1, 1), padding='same')(y)
x = BatchNormalization()(x)
x = add([x, y])
x = Activation('relu')(x)
x = UpSampling3D((2, 2))(x)
x = Conv3D(num_channels, (3, 3), activation='linear', padding='same')(x)
# here is a second stage decode chance
for _ in range(2):
x = inception_module(x, ds=ds)
x = BatchNormalization()(x)
y = Activation('relu')(x)
x = inception_module(x, ds=ds)
x = BatchNormalization()(x)
x = add([x, y])
decoded = Conv3D(num_channels, (3, 3), activation='linear',
padding='same')(x)
vae = Model(input_img, decoded)
def vae_loss(y_true, y_pred):
xent_loss = metrics.mean_squared_error(K.flatten(y_true),
K.flatten(y_pred))
kl_loss = -0.5 * K.sum(
1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return K.mean(xent_loss + kl_loss)
vae.compile(optimizer=Adam(lr=learning_rate), loss=vae_loss)
print(vae.summary())
json_string = vae.to_json()
with open(model_path, 'w') as f:
json.dump(json_string, f)
return vae
