def green_block(inp, filters, data_format='channels_first', name=None):
"""
green_block(inp, filters, name=None)
------------------------------------
Implementation of the special residual block used in the paper. The block
consists of two (GroupNorm --> ReLu --> 3x3x3 non-strided Convolution)
units, with a residual connection from the input `inp` to the output. Used
internally in the model. Can be used independently as well.
Parameters
----------
`inp`: An keras.layers.layer instance, required
The keras layer just preceding the green block.
`filters`: integer, required
No. of filters to use in the 3D convolutional block. The output
layer of this green block will have this many no. of channels.
`data_format`: string, optional
The format of the input data. Must be either 'chanels_first' or
'channels_last'. Defaults to `channels_first`, as used in the paper.
`name`: string, optional
The name to be given to this green block. Defaults to None, in which
case, keras uses generated names for the involved layers. If a string
is provided, the names of individual layers are generated by attaching
a relevant prefix from [GroupNorm_, Res_, Conv3D_, Relu_, ], followed
by _1 or _2.
Returns
-------
`out`: A keras.layers.Layer instance
The output of the green block. Has no. of channels equal to `filters`.
The size of the rest of the dimensions remains same as in `inp`.
"""
inp_res = Conv3D(filters=filters,
kernel_size=(1, 1, 1),
strides=1,
data_format=data_format,
name=f'Res_{name}' if name else None)(inp)
# axis=1 for channels_first data format
# No. of groups = 8, as given in the paper
x = GroupNormalization(groups=8,
axis=1 if data_format == 'channels_first' else 0,
name=f'GroupNorm_1_{name}' if name else None)(inp)
x = Activation('relu', name=f'Relu_1_{name}' if name else None)(x)
x = Conv3D(filters=filters,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format=data_format,
name=f'Conv3D_1_{name}' if name else None)(x)
x = GroupNormalization(groups=8,
axis=1 if data_format == 'channels_first' else 0,
name=f'GroupNorm_2_{name}' if name else None)(x)
x = Activation('relu', name=f'Relu_2_{name}' if name else None)(x)
x = Conv3D(filters=filters,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format=data_format,
name=f'Conv3D_2_{name}' if name else None)(x)
out = Add(name=f'Out_{name}' if name else None)([x, inp_res])
return out
def generator_block(self, name):
"""Creates a generator model"""
# generator same dim as input and output if multiple of 4
inputs = Input(shape=(1, self.image_row, self.image_column,
self.image_depth))
# Representation
gennet = ReflectPadding3D(padding=3)(inputs)
gennet = Conv3D(self.generator_kernel,
7,
strides=1,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_conv1',
data_format='channels_first')(gennet)
gennet = InstanceNormalization3D(name=name +
'_gen_isnorm_conv1')(gennet)
gennet = Activation('relu')(gennet)
# Downsampling 1
gennet = ReflectPadding3D(padding=1)(gennet)
gennet = Conv3D(self.generator_kernel * 2,
3,
strides=2,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_conv2',
data_format='channels_first')(gennet)
gennet = InstanceNormalization3D(name=name +
'_gen_isnorm_conv2')(gennet)
gennet = Activation('relu')(gennet)
# Downsampling 2
gennet = ReflectPadding3D(padding=1)(gennet)
gennet = Conv3D(self.generator_kernel * 4,
3,
strides=2,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_conv3',
data_format='channels_first')(gennet)
gennet = InstanceNormalization3D(name=name +
'_gen_isnorm_conv3')(gennet)
gennet = Activation('relu')(gennet)
# Resnet blocks : 6, 8*4 = 32
gennet = resnet_blocks(gennet,
self.generator_kernel * 4,
name=name + '_gen_block1')
gennet = resnet_blocks(gennet,
self.generator_kernel * 4,
name=name + '_gen_block2')
gennet = resnet_blocks(gennet,
self.generator_kernel * 4,
name=name + '_gen_block3')
gennet = resnet_blocks(gennet,
self.generator_kernel * 4,
name=name + '_gen_block4')
gennet = resnet_blocks(gennet,
self.generator_kernel * 4,
name=name + '_gen_block5')
gennet = resnet_blocks(gennet,
self.generator_kernel * 4,
name=name + '_gen_block6')
# Upsampling 1
gennet = UpSampling3D(size=(2, 2, 2),
data_format='channels_first')(gennet)
gennet = ReflectPadding3D(padding=1)(gennet)
gennet = Conv3D(self.generator_kernel * 2,
3,
strides=1,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_deconv1',
data_format='channels_first')(gennet)
gennet = InstanceNormalization3D(name=name +
'_gen_isnorm_deconv1')(gennet)
gennet = Activation('relu')(gennet)
# Upsampling 2
gennet = UpSampling3D(size=(2, 2, 2),
data_format='channels_first')(gennet)
gennet = ReflectPadding3D(padding=1)(gennet)
gennet = Conv3D(self.generator_kernel,
3,
strides=1,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_deconv2',
data_format='channels_first')(gennet)
gennet = InstanceNormalization3D(name=name +
'_gen_isnorm_deconv2')(gennet)
gennet = Activation('relu')(gennet)
# Reconstruction
gennet = ReflectPadding3D(padding=3)(gennet)
gennet = Conv3D(2,
7,
strides=1,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_1conv',
data_format='channels_first')(gennet)
predictions = gennet
predictions = activation_SegSRGAN(is_residual=self.is_residual)(
[predictions, inputs]) # sigmoid proba + add input and pred SR
model = Model(inputs=inputs, outputs=predictions, name=name)
return model
def generator_block_conditionnal(
self,
name): # generateur meme dim en entree et sortie si multiple de 4
#
im = Input(shape=(1, self.image_row, self.image_column,
self.image_depth),
name='dis_input')
res = Input(shape=(1, self.image_row, self.image_column,
self.image_depth),
name='dis_input_res')
inputs = Concatenate(axis=-4)([im, res])
# Representation
gennet = ReflectPadding3D(padding=3)(inputs)
gennet = Conv3D(self.generator_kernel,
7,
strides=1,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_conv1',
data_format='channels_first')(gennet)
gennet = InstanceNormalization3D(name=name +
'_gen_isnorm_conv1')(gennet)
gennet = Activation('relu')(gennet)
# Downsampling 1
gennet = ReflectPadding3D(padding=1)(gennet)
gennet = Conv3D(self.generator_kernel * 2,
3,
strides=2,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_conv2',
data_format='channels_first')(gennet)
gennet = InstanceNormalization3D(name=name +
'_gen_isnorm_conv2')(gennet)
gennet = Activation('relu')(gennet)
# Downsampling 2
gennet = ReflectPadding3D(padding=1)(gennet)
gennet = Conv3D(self.generator_kernel * 4,
3,
strides=2,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_conv3',
data_format='channels_first')(gennet)
gennet = InstanceNormalization3D(name=name +
'_gen_isnorm_conv3')(gennet)
gennet = Activation('relu')(gennet)
# Resnet blocks : 6, 8*4 = 32
gennet = resnet_blocks(gennet,
self.generator_kernel * 4,
name=name + '_gen_block1')
gennet = resnet_blocks(gennet,
self.generator_kernel * 4,
name=name + '_gen_block2')
gennet = resnet_blocks(gennet,
self.generator_kernel * 4,
name=name + '_gen_block3')
gennet = resnet_blocks(gennet,
self.generator_kernel * 4,
name=name + '_gen_block4')
gennet = resnet_blocks(gennet,
self.generator_kernel * 4,
name=name + '_gen_block5')
gennet = resnet_blocks(gennet,
self.generator_kernel * 4,
name=name + '_gen_block6')
# Upsampling 1
gennet = UpSampling3D(size=(2, 2, 2),
data_format='channels_first')(gennet)
gennet = ReflectPadding3D(padding=1)(gennet)
gennet = Conv3D(self.generator_kernel * 2,
3,
strides=1,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_deconv1',
data_format='channels_first')(gennet)
gennet = InstanceNormalization3D(name=name +
'_gen_isnorm_deconv1')(gennet)
gennet = Activation('relu')(gennet)
# Upsampling 2
gennet = UpSampling3D(size=(2, 2, 2),
data_format='channels_first')(gennet)
gennet = ReflectPadding3D(padding=1)(gennet)
gennet = Conv3D(self.generator_kernel,
3,
strides=1,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_deconv2',
data_format='channels_first')(gennet)
gennet = InstanceNormalization3D(name=name +
'_gen_isnorm_deconv2')(gennet)
gennet = Activation('relu')(gennet)
# Reconstruction
gennet = ReflectPadding3D(padding=3)(gennet)
gennet = Conv3D(2,
7,
strides=1,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_1conv',
data_format='channels_first')(gennet)
predictions = gennet
predictions = activation_SegSRGAN(is_residual=self.is_residual)(
[predictions, im]) # sigmoid proba + add input and pred SR
model = Model(inputs=[im, res], outputs=predictions, name=name)
return model
def build_generator(self):
"""U-Net Generator"""
self.stack_downsamling = []
def conv3d(layer_input, filters, f_size=4, bn=True):
"""Layers used during downsampling"""
# # TODO: einziger Weg wie man vor dem conv. das padding selbst bestimmen kann
# padded_input = tf.pad(input, [[0, 0], [2, 2], [1, 1], [0, 0]], "CONSTANT")
# output = tf.nn.conv2d(padded_input, filter, strides=[1, 1, 1, 1], padding="VALID")
d = Conv3D(filters, kernel_size=f_size, strides=2,
padding='same')(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if bn:
d = BatchNormalization(momentum=0.8)(d)
self.stack_downsamling.append(int(d.shape[3]))
print('downsampling:\t\t\t', d.shape)
return d
def deconv3d(layer_input,
skip_input,
filters,
cnt,
f_size=4,
dropout_rate=0):
"""Layers used during upsampling"""
# TODO: ist das hier genug nur auf den vorletzten zu überprüfen?? generisch schreiben?
if self.stack_downsamling[cnt - 1] == 1:
u = UpSampling3D(size=(2, 2, 1),
data_format="channels_last")(layer_input)
else:
u = UpSampling3D(size=(2, 2, 2),
data_format="channels_last")(layer_input)
u = Conv3D(filters,
kernel_size=f_size,
strides=1,
padding='same',
activation='relu')(u)
if dropout_rate:
u = Dropout(dropout_rate)(u)
u = BatchNormalization(momentum=0.8)(u)
print('upsampling:\t\t\t', u.shape)
u = Concatenate()([u, skip_input])
return u
# Image input (distinguish for two potency stack number)
d0 = Input(shape=self.vol_shape)
print('generator-model input:\t\t', d0.shape)
self.stack_downsamling.append(int(d0.shape[3]))
c = 0
# Downsampling
d1 = conv3d(d0, self.gf, bn=False)
c += 1
d2 = conv3d(d1, self.gf * 2)
c += 1
d3 = conv3d(d2, self.gf * 4)
c += 1
d4 = conv3d(d3, self.gf * 8)
c += 1
d5 = conv3d(d4, self.gf * 8)
c += 1
# d6 = conv3d(d5, self.gf*8); c += 1
# d7 = conv3d(d6, self.gf*8); c += 1
# Upsampling
# u1 = deconv3d(d7, d6, self.gf*8, cnt=c); c -= 1
# u2 = deconv3d(u1, d5, self.gf*8, cnt=c); c -= 1
# u3 = deconv3d(u2, d4, self.gf*8, cnt=c); c -= 1
u3 = deconv3d(d5, d4, self.gf * 8, cnt=c)
c -= 1
u4 = deconv3d(u3, d3, self.gf * 4, cnt=c)
c -= 1
u5 = deconv3d(u4, d2, self.gf * 2, cnt=c)
c -= 1
u6 = deconv3d(u5, d1, self.gf, cnt=c)
c -= 1
u7 = UpSampling3D(size=2, data_format="channels_last")(u6)
output_vol = Conv3D(self.channels,
kernel_size=4,
strides=1,
padding='same',
activation='tanh')(u7)
print('generator-model output:\t\t', d0.shape, output_vol.shape)
return Model(d0, output_vol, name='generator')
def create_model_3dcnn(input_shape=None,
n_classes=None,
n_filters=None,
kernel_sizes=None,
n_convolutions_per_block=None,
pooling_size=None,
strides=None,
size_last_layer=None,
actFunc='relu',
learning_rate=0.001,
dropout_rate=0,
batch_normalization=False,
nb_epoch=200,
loss='categorical_crossentropy',
metrics=None,
optimizer='adam',
gpu_device='/gpu:0'):
# size_last_layer
# n_filters
# kernel_size
# strides
# n_convolutions_per_block
if input_shape is None:
input_shape = []
if n_classes is None:
n_classes = []
if n_filters is None:
n_filters = []
if kernel_sizes is None:
kernel_sizes = []
if n_convolutions_per_block is None:
n_convolutions_per_block = []
if pooling_size is None:
pooling_size = []
if strides is None:
strides = []
if size_last_layer is None:
size_last_layer = []
if metrics is None:
metrics = ['accuracy']
ks = kernel_sizes
ps = pooling_size
nf = n_filters
input_shape = (input_shape[0], input_shape[1], input_shape[2], 1)
model = Sequential()
for ind_blocks in range(len(n_filters)):
for ind_convs in range(n_convolutions_per_block):
if ind_blocks == 0 and ind_convs == 0:
with tf.device(gpu_device):
model.add(
Conv3D(nf[ind_blocks], [ks[0], ks[1], ks[2]],
strides=strides,
padding='same',
input_shape=input_shape))
model.add(Activation(actFunc))
if batch_normalization:
model.add(BatchNormalization())
else:
with tf.device(gpu_device):
model.add(
Conv3D(nf[ind_blocks], [ks[0], ks[1], ks[2]],
strides=strides,
padding='same'))
model.add(Activation(actFunc))
if batch_normalization:
model.add(BatchNormalization())
with tf.device(gpu_device):
if ps:
model.add(MaxPooling3D(pool_size=(ps, ps, ps)))
if dropout_rate:
model.add(Dropout(dropout_rate))
with tf.device(gpu_device):
model.add(Flatten())
model.add(Dense(size_last_layer))
model.add(Activation(actFunc))
if batch_normalization:
model.add(BatchNormalization())
with tf.device(gpu_device):
model.add(Dense(n_classes))
model.add(Activation('softmax'))
# initiate RMSprop optimizer
optimizer = define_optimizer(optimizer_type=optimizer, lr=learning_rate)
# Let's train the model using RMSprop
model.compile(loss=loss, optimizer=optimizer, metrics=metrics)
print(model.summary())
return model
def dk(self, x, k):
x = Conv3D(filters=k, kernel_size=3, strides=2, padding='same')(x)
x = self.normalization(axis=3, center=True,
epsilon=1e-5)(x, training=True)
x = Activation('relu')(x)
return x
def build_generator(self, number_of_filters_per_layer=(128, 64), kernel_size=3):
model = Sequential()
# To build the generator, we create the reverse encoder model
# and simply build the reverse model
encoder = None
if self.dimensionality == 2:
autoencoder, encoder = create_convolutional_autoencoder_model_2d(
input_image_size=self.input_image_size,
number_of_filters_per_layer=(*(number_of_filters_per_layer[::-1]), self.latent_dimension),
convolution_kernel_size=(5, 5),
deconvolution_kernel_size=(5, 5))
else:
autoencoder, encoder = create_convolutional_autoencoder_model_3d(
input_image_size=self.input_image_size,
number_of_filters_per_layer=(*(number_of_filters_per_layer[::-1]), self.latent_dimension),
convolution_kernel_size=(5, 5, 5),
deconvolution_kernel_size=(5, 5, 5))
encoder_layers = encoder.layers
penultimate_layer = encoder_layers[len(encoder_layers) - 2]
model.add(Dense(units=penultimate_layer.output_shape[1],
input_dim=self.latent_dimension,
activation="relu"))
conv_layer = encoder_layers[len(encoder_layers) - 3]
resampled_size = conv_layer.output_shape[1:(self.dimensionality + 2)]
model.add(Reshape(resampled_size))
count = 0
for i in range(len(encoder_layers) - 3, 1, -1):
conv_layer = encoder_layers[i]
resampled_size = conv_layer.output_shape[1:(self.dimensionality + 1)]
if self.dimensionality == 2:
model.add(ResampleTensorLayer2D(shape=resampled_size,
interpolation_type='linear'))
model.add(Conv2D(filters=number_of_filters_per_layer[count],
kernel_size=kernel_size,
padding='same'))
else:
model.add(ResampleTensorLayer3D(shape=resampled_size,
interpolation_type='linear'))
model.add(Conv3D(filters=number_of_filters_per_layer[count],
kernel_size=kernel_size,
padding='same'))
model.add(BatchNormalization(momentum=0.8))
model.add(Activation(activation='relu'))
count += 1
number_of_channels = self.input_image_size[-1]
spatial_dimensions = self.input_image_size[:self.dimensionality]
if self.dimensionality == 2:
model.add(ResampleTensorLayer2D(shape=spatial_dimensions,
interpolation_type='linear'))
model.add(Conv2D(filters=number_of_channels,
kernel_size=kernel_size,
padding='same'))
else:
model.add(ResampleTensorLayer3D(shape=spatial_dimensions,
interpolation_type='linear'))
model.add(Conv3D(filters=number_of_channels,
kernel_size=kernel_size,
padding='same'))
model.add(Activation(activation="tanh"))
noise = Input(shape=(self.latent_dimension,))
image = model(noise)
generator = Model(inputs=noise, outputs=image)
return(generator)
y = tf.reshape(y, [batch_size, d_z, d_height, d_width, 512])
return y
#def space_to_depth_x2(x):
# return tf.nn.space_to_depth(x, block_size=2)
input_image = Input(shape=(IMAGE_Z, IMAGE_H, IMAGE_W, 1))
true_boxes = Input(shape=(1, 1, 1, 1, TRUE_BOX_BUFFER,
6)) # ?,?,?, #, x,y,z,h,w,d
dropout_rate = 0.2
# Layer 1
x = Conv3D(32, (3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='conv_1',
use_bias=False)(input_image)
x = BatchNormalization(name='norm_1')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling3D(pool_size=(2, 2, 2))(x)
Dropout(dropout_rate)(x)
# Layer 2
x = Conv3D(64, (3, 3, 3),
strides=(1, 1, 1),
padding='same',
name='conv_2',
use_bias=False)(x)
x = BatchNormalization(name='norm_2')(x)
x = LeakyReLU(alpha=0.1)(x)
def unet_model():
inputs = Input(input_shape)
conv1 = Conv3D(32, 3, 3, 3, activation='relu', border_mode='same')(inputs)
conv1 = Conv3D(32, 3, 3, 3, activation='relu', border_mode='same')(conv1)
pool1 = MaxPooling3D(pool_size=pool_size)(conv1)
conv2 = Conv3D(64, 3, 3, 3, activation='relu', border_mode='same')(pool1)
conv2 = Conv3D(64, 3, 3, 3, activation='relu', border_mode='same')(conv2)
pool2 = MaxPooling3D(pool_size=pool_size)(conv2)
conv3 = Conv3D(128, 3, 3, 3, activation='relu', border_mode='same')(pool2)
conv3 = Conv3D(128, 3, 3, 3, activation='relu', border_mode='same')(conv3)
pool3 = MaxPooling3D(pool_size=pool_size)(conv3)
conv4 = Conv3D(256, 3, 3, 3, activation='relu', border_mode='same')(pool3)
conv4 = Conv3D(256, 3, 3, 3, activation='relu', border_mode='same')(conv4)
pool4 = MaxPooling3D(pool_size=pool_size)(conv4)
conv5 = Conv3D(512, 3, 3, 3, activation='relu', border_mode='same')(pool4)
conv5 = Conv3D(512, 3, 3, 3, activation='relu', border_mode='same')(conv5)
up6 = merge([UpSampling3D(size=pool_size)(conv5), conv4], mode='concat', concat_axis=1)
conv6 = Conv3D(256, 3, 3, 3, activation='relu', border_mode='same')(up6)
conv6 = Conv3D(256, 3, 3, 3, activation='relu', border_mode='same')(conv6)
up7 = merge([UpSampling3D(size=pool_size)(conv6), conv3], mode='concat', concat_axis=1)
conv7 = Conv3D(128, 3, 3, 3, activation='relu', border_mode='same')(up7)
conv7 = Conv3D(128, 3, 3, 3, activation='relu', border_mode='same')(conv7)
up8 = merge([UpSampling3D(size=pool_size)(conv7), conv2], mode='concat', concat_axis=1)
conv8 = Conv3D(64, 3, 3, 3, activation='relu', border_mode='same')(up8)
conv8 = Conv3D(64, 3, 3, 3, activation='relu', border_mode='same')(conv8)
up9 = merge([UpSampling3D(size=pool_size)(conv8), conv1], mode='concat', concat_axis=1)
conv9 = Conv3D(32, 3, 3, 3, activation='relu', border_mode='same')(up9)
conv9 = Conv3D(32, 3, 3, 3, activation='relu', border_mode='same')(conv9)
conv10 = Conv3D(n_labels, 1, 1, 1)(conv9)
act = Activation('sigmoid')(conv10)
model = Model(input=inputs, output=act)
model.compile(optimizer=Adam(lr=1e-5), loss=dice_coef_loss, metrics=[dice_coef])
return model
#
# Where $i$ is the current depth.
#
# So at depth $i=0$:
# $$filters_{0} = 32 \times (2^{0}) = 32$$
#
# ### Layer 0
# There are two convolutional layers for each depth
# Run the next cell to create the first 3D convolution
# In[4]:
# Define a Conv3D tensor with 32 filters
down_depth_0_layer_0 = Conv3D(filters=32,
kernel_size=(3, 3, 3),
padding='same',
strides=(1, 1, 1))(input_layer)
down_depth_0_layer_0
# Notice that with 32 filters, the result you get above is a tensor with 32 channels.
#
# Run the next cell to add a relu activation to the first convolutional layer
# In[5]:
# Add a relu activation to layer 0 of depth 0
down_depth_0_layer_0 = Activation('relu')(down_depth_0_layer_0)
down_depth_0_layer_0
# ### Depth 0, Layer 1
# For layer 1 of depth 0, the formula for calculating the number of filters is:
xtest = xtest.reshape(x_test.shape[0], 16, 16, 16, 3)
# Label变成One-Hot的
## convert target variable into one-hot
y_train = keras.utils.to_categorical(y_train, 10)
y_test = keras.utils.to_categorical(y_test, 10)
# (10000,10)
print(y_train.shape)
# 搭建神经网络结构
## input layer
input_layer = Input((16, 16, 16, 3))
## convolutional layers
conv_layer1 = Conv3D(filters=8, kernel_size=(3, 3, 3),
activation='relu')(input_layer)
conv_layer2 = Conv3D(filters=16, kernel_size=(3, 3, 3),
activation='relu')(conv_layer1)
## add max pooling to obtain the most imformatic features
pooling_layer1 = MaxPool3D(pool_size=(2, 2, 2))(conv_layer2)
conv_layer3 = Conv3D(filters=32, kernel_size=(3, 3, 3),
activation='relu')(pooling_layer1)
conv_layer4 = Conv3D(filters=64, kernel_size=(3, 3, 3),
activation='relu')(conv_layer3)
pooling_layer2 = MaxPool3D(pool_size=(2, 2, 2))(conv_layer4)
## perform batch normalization on the convolution outputs before feeding it to MLP architecture
pooling_layer2 = BatchNormalization()(pooling_layer2)
flatten_layer = Flatten()(pooling_layer2)
def unet_model_3d(loss_function,
input_shape=(4, 160, 160, 16),
pool_size=(2, 2, 2),
n_labels=3,
initial_learning_rate=0.00001,
deconvolution=False,
depth=4,
n_base_filters=32,
include_label_wise_dice_coefficients=False,
metrics=[],
batch_normalization=False,
activation_name="sigmoid"):
"""
Builds the 3D UNet Keras model.f
:param metrics: List metrics to be calculated during model training (default is dice coefficient).
:param include_label_wise_dice_coefficients: If True and n_labels is greater than 1, model will report the dice
coefficient for each label as metric.
:param n_base_filters: The number of filters that the first layer in the convolution network will have. Following
layers will contain a multiple of this number. Lowering this number will likely reduce the amount of memory required
to train the model.
:param depth: indicates the depth of the U-shape for the model. The greater the depth, the more max pooling
layers will be added to the model. Lowering the depth may reduce the amount of memory required for training.
:param input_shape: Shape of the input data (n_chanels, x_size, y_size, z_size). The x, y, and z sizes must be
divisible by the pool size to the power of the depth of the UNet, that is pool_size^depth.
:param pool_size: Pool size for the max pooling operations.
:param n_labels: Number of binary labels that the model is learning.
:param initial_learning_rate: Initial learning rate for the model. This will be decayed during training.
:param deconvolution: If set to True, will use transpose convolution(deconvolution) instead of up-sampling. This
increases the amount memory required during training.
:return: Untrained 3D UNet Model
"""
inputs = Input(input_shape)
current_layer = inputs
levels = list()
# add levels with max pooling
for layer_depth in range(depth):
layer1 = create_convolution_block(
input_layer=current_layer,
n_filters=n_base_filters * (2**layer_depth),
batch_normalization=batch_normalization)
layer2 = create_convolution_block(
input_layer=layer1,
n_filters=n_base_filters * (2**layer_depth) * 2,
batch_normalization=batch_normalization)
if layer_depth < depth - 1:
current_layer = MaxPooling3D(pool_size=pool_size)(layer2)
levels.append([layer1, layer2, current_layer])
else:
current_layer = layer2
levels.append([layer1, layer2])
# add levels with up-convolution or up-sampling
for layer_depth in range(depth - 2, -1, -1):
up_convolution = get_up_convolution(
pool_size=pool_size,
deconvolution=deconvolution,
n_filters=current_layer._keras_shape[1])(current_layer)
concat = concatenate([up_convolution, levels[layer_depth][1]], axis=1)
current_layer = create_convolution_block(
n_filters=levels[layer_depth][1]._keras_shape[1],
input_layer=concat,
batch_normalization=batch_normalization)
current_layer = create_convolution_block(
n_filters=levels[layer_depth][1]._keras_shape[1],
input_layer=current_layer,
batch_normalization=batch_normalization)
final_convolution = Conv3D(n_labels, (1, 1, 1))(current_layer)
act = Activation(activation_name)(final_convolution)
model = Model(inputs=inputs, outputs=act)
if not isinstance(metrics, list):
metrics = [metrics]
model.compile(optimizer=Adam(lr=initial_learning_rate),
loss=loss_function,
metrics=metrics)
return model
def GetEncoder3D(img_shape, arm_size, gripper_size, dropout_rate,
filters, tile=False, dropout=True, leaky=True,
pre_tiling_layers=0,
post_tiling_layers=2,
kernel_size=[3,3,3],
time_distributed=10,):
arm_in = Input((time_distributed, arm_size,))
gripper_in = Input((time_distributed, gripper_size,))
height4 = img_shape[1]/4
width4 = img_shape[2]/4
height2 = img_shape[1]/2
width2 = img_shape[2]/2
height = img_shape[1]
width = img_shape[2]
channels = img_shape[3]
samples = Input(shape=img_shape)
'''
This is set up to use 3D convolutions to operate over a bunch of temporally
grouped frames. The assumption is that this will allow us to capture
temporal dependencies between actions better than we otherwise would be
able to.
'''
if leaky:
relu = lambda: LeakyReLU(alpha=0.2)
else:
relu = lambda: Activation('relu')
x = samples
for i in range(pre_tiling_layers):
x = Conv3D(filters,
kernel_size=kernel_size,
strides=(1, 2, 2),
padding='same')(x)
x = relu()(x)
if dropout:
x = Dropout(dropout_rate)(x)
# ===============================================
# ADD TILING
if tile:
tile_width = int(width/(pre_tiling_layers+1))
tile_height = int(height/(pre_tiling_layers+1))
x, _ = TileArmAndGripper(arm_in, gripper_in,
tile_width, tile_height,
time_distributed)
ins = [samples, arm_in, gripper_in]
else:
ins = [samples]
for i in range(post_tiling_layers):
x = Conv3D(filters,
kernel_size=kernel_size,
strides=(2, 2, 2),
padding='same')(x)
x = relu()(x)
if dropout:
x = Dropout(dropout_rate)(x)
return ins, x
# faire un changement dans la dimension des elements d'apprentissage et de test pour qu'on puisse les utiliser dans l'entree de notre reseau de neurone
x_train = x_train.reshape(len(x_train), 91, 109, 91, 1)
x_test = x_test.reshape(len(x_test), 91, 109, 91, 1)
layer_size=256
# le nom de dossier dans lequel on va trouver les resultats bien detailler de l'execution de notre programme
NAME="{} conv_layer {} layer_size {} dense_layer {}".format(2,layer_size,3,int(time.time()))
tensorboard=TensorBoard(log_dir='testddddddddd2_conv_180ep_lr0.0001_8542b20all_data/{}'.format(NAME))
#(kernel_regularizer=regularizers.l2(0.01),activity_regularizer=regularizers.l1(0.01))
# maintenant on commence a construire notre reseau de neurone convolutif
# Dans cet exemple on an fait deux couches convolutifs
model=Sequential()
model.add(Conv3D(layer_size, (3,3,3), padding = 'same', activation='relu', use_bias = False, input_shape=(91, 109, 91, 1)))# la premiere couche
model.add(MaxPooling3D(pool_size=(2, 2, 2)))
model.add(Dropout(0.25)) # on ajoute ce ligne apres chaque couche pour aider le "loss" de notre programme converger (il n'est pas forcement de faire ce ligne dans un reseau de neurone)
model.add(Conv3D(128, (3,3,3), padding = 'same', use_bias = False, activation='relu'))
model.add(MaxPooling3D(pool_size=(2, 2, 2)))
model.add(Dropout(0.3))
# apres deux couches convolutifs on va faire un reseau compltement connectee
model.add(Flatten()) # cette couche a permet de convertir l'image a un vecteur colone
model.add(Dropout(0.35))
model.add(Dense(512, activation='relu'))
model.add(Dense(128,input_dim=128,activity_regularizer=regularizers.l1(0.01)))# dans cette couche cachee on a utilise la methode de regularization l1
model.add(Dense(1, activation='sigmoid')) # on a utilise la fonction d'activation sigmoid parce qu'il y a une sortie binaire
sgd = SGD(lr = 0.0001, decay = 1e-6, momentum = 0.9, nesterov = True) # ce sont les parametre de l'optimiseur
model.compile(loss = 'binary_crossentropy', optimizer='sgd', metrics = ['accuracy']) # choisir les metrics et l'optimiseur qu'on va utiliser dans notre reseau de neurone
def build_3d_cnn(w, h, d, s, num_outputs):
from keras.layers import Input, Dense
from keras.models import Sequential
from keras.layers import Conv3D, MaxPooling3D, Reshape, BatchNormalization
from keras.layers import Activation, Dropout, Flatten, Cropping3D
#Credit: https://github.com/jessecha/DNRacing/blob/master/3D_CNN_Model/model.py
'''
w : width
h : height
d : depth
s : n_stacked
'''
input_shape = (s, h, w, d)
model = Sequential()
#First layer
model.add(
Cropping3D(cropping=((0, 0), (50, 10), (0, 0)),
input_shape=input_shape)) #trim pixels off top
# Second layer
model.add(
Conv3D(filters=16,
kernel_size=(3, 3, 3),
strides=(1, 3, 3),
data_format='channels_last',
padding='same'))
model.add(Activation('relu'))
model.add(
MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
padding='valid',
data_format=None))
# Third layer
model.add(
Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
data_format='channels_last',
padding='same'))
model.add(Activation('relu'))
model.add(
MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
padding='valid',
data_format=None))
# Fourth layer
model.add(
Conv3D(filters=64,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
data_format='channels_last',
padding='same'))
model.add(Activation('relu'))
model.add(
MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
padding='valid',
data_format=None))
# Fifth layer
model.add(
Conv3D(filters=128,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
data_format='channels_last',
padding='same'))
model.add(Activation('relu'))
model.add(
MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
padding='valid',
data_format=None))
# Fully connected layer
model.add(Flatten())
model.add(Dense(256))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(256))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(num_outputs))
#model.add(Activation('tanh'))
return model
def build_model(self,
img_shape=(32, 168, 168),
learning_rate=5e-5,
gpu_id=None,
nb_gpus=None):
input_img = Input((*img_shape, 1), name='img_inp')
unsupervised_label = Input((*img_shape, 5), name='unsup_label_inp')
# supervised_flag = Input(shape=img_shape, name='flag_inp')
kernel_init = 'he_normal'
sfs = 16 # start filter size
bn = True
do = True
conv1, conv1_b_m = self.downLayer(input_img, sfs, 1, bn)
conv2, conv2_b_m = self.downLayer(conv1, sfs * 2, 2, bn)
conv3 = Conv3D(sfs * 4, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer=kernel_init,
name='conv' + str(3) + '_1')(conv2)
if bn:
conv3 = BatchNormalization()(conv3)
conv3 = Conv3D(sfs * 8, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer=kernel_init,
name='conv' + str(3) + '_2')(conv3)
if bn:
conv3 = BatchNormalization()(conv3)
pool3 = MaxPooling3D(pool_size=(2, 2, 2))(conv3)
# conv3, conv3_b_m = downLayer(conv2, sfs*4, 3, bn)
conv4 = Conv3D(sfs * 16, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer=kernel_init,
name='conv4_1')(pool3)
if bn:
conv4 = BatchNormalization()(conv4)
if do:
conv4 = Dropout(0.5, seed=4, name='Dropout_' + str(4))(conv4)
conv4 = Conv3D(sfs * 16, (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer=kernel_init,
name='conv4_2')(conv4)
if bn:
conv4 = BatchNormalization()(conv4)
# conv5 = upLayer(conv4, conv3_b_m, sfs*16, 5, bn, do)
up1 = Conv3DTranspose(sfs * 16, (2, 2, 2),
strides=(2, 2, 2),
activation='relu',
padding='same',
name='up' + str(5))(conv4)
up1 = concatenate([up1, conv3])
conv5 = Conv3D(int(sfs * 8), (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer=kernel_init,
name='conv' + str(5) + '_1')(up1)
if bn:
conv5 = BatchNormalization()(conv5)
if do:
conv5 = Dropout(0.5, seed=5, name='Dropout_' + str(5))(conv5)
conv5 = Conv3D(int(sfs * 8), (3, 3, 3),
activation='relu',
padding='same',
kernel_initializer=kernel_init,
name='conv' + str(5) + '_2')(conv5)
if bn:
conv5 = BatchNormalization()(conv5)
conv6 = self.upLayer(conv5, conv2_b_m, sfs * 8, 6, bn, do)
conv7 = self.upLayer(conv6, conv1_b_m, sfs * 4, 7, bn, do)
conv_out = Conv3D(5, (1, 1, 1), name='conv_final_softmax')(conv7)
# conv_out = Lambda(lambda x: x / temp, name='scaling')(conv_out)
conv_out_sm = Activation('softmax')(conv_out)
pz_sm_out = Lambda(lambda x: x[:, :, :, :, 0], name='pz')(conv_out_sm)
cz_sm_out = Lambda(lambda x: x[:, :, :, :, 1], name='cz')(conv_out_sm)
us_sm_out = Lambda(lambda x: x[:, :, :, :, 2], name='us')(conv_out_sm)
afs_sm_out = Lambda(lambda x: x[:, :, :, :, 3],
name='afs')(conv_out_sm)
bg_sm_out = Lambda(lambda x: x[:, :, :, :, 4], name='bg')(conv_out_sm)
pz_ensemble_pred = Lambda(lambda x: x[:, :, :, :, 0],
name='pzu')(unsupervised_label)
cz_ensemble_pred = Lambda(lambda x: x[:, :, :, :, 1],
name='czu')(unsupervised_label)
us_ensemble_pred = Lambda(lambda x: x[:, :, :, :, 2],
name='usu')(unsupervised_label)
afs_ensemble_pred = Lambda(lambda x: x[:, :, :, :, 3],
name='afsu')(unsupervised_label)
bg_ensemble_pred = Lambda(lambda x: x[:, :, :, :, 4],
name='bgu')(unsupervised_label)
optimizer = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999)
if (nb_gpus is None):
p_model = Model(
[input_img, unsupervised_label],
[pz_sm_out, cz_sm_out, us_sm_out, afs_sm_out, bg_sm_out])
p_model.compile(
optimizer=optimizer,
loss={
'pz':
self.semi_supervised_loss(pz_ensemble_pred,
unsup_loss_class_wt=1),
'cz':
self.semi_supervised_loss(cz_ensemble_pred, 1),
'us':
self.semi_supervised_loss(us_ensemble_pred, 2),
'afs':
self.semi_supervised_loss(afs_ensemble_pred, 2),
'bg':
self.semi_supervised_loss(bg_ensemble_pred, 1)
},
metrics={
'pz':
[self.dice_coef,
self.unsup_dice_tb(pz_ensemble_pred, 1)],
'cz':
[self.dice_coef,
self.unsup_dice_tb(cz_ensemble_pred, 1)],
'us':
[self.dice_coef,
self.unsup_dice_tb(us_ensemble_pred, 2)],
'afs':
[self.dice_coef,
self.unsup_dice_tb(afs_ensemble_pred, 2)],
'bg':
[self.dice_coef,
self.unsup_dice_tb(bg_ensemble_pred, 1)]
})
else:
with tf.device(gpu_id):
model = Model(
[input_img, unsupervised_label],
[pz_sm_out, cz_sm_out, us_sm_out, afs_sm_out, bg_sm_out])
p_model = multi_gpu_model(model, gpus=nb_gpus)
p_model.compile(
optimizer=optimizer,
loss={
'pz':
self.semi_supervised_loss(pz_ensemble_pred,
unsup_loss_class_wt=1),
'cz':
self.semi_supervised_loss(cz_ensemble_pred, 1),
'us':
self.semi_supervised_loss(us_ensemble_pred, 2),
'afs':
self.semi_supervised_loss(afs_ensemble_pred, 2),
'bg':
self.semi_supervised_loss(bg_ensemble_pred, 1)
},
metrics={
'pz': [
self.dice_coef,
self.unsup_dice_tb(pz_ensemble_pred, 1)
],
'cz': [
self.dice_coef,
self.unsup_dice_tb(cz_ensemble_pred, 1)
],
'us': [
self.dice_coef,
self.unsup_dice_tb(us_ensemble_pred, 2)
],
'afs': [
self.dice_coef,
self.unsup_dice_tb(afs_ensemble_pred, 2)
],
'bg': [
self.dice_coef,
self.unsup_dice_tb(bg_ensemble_pred, 1)
]
},
loss_weights={
'pz': 1,
'cz': 1,
'us': 2,
'afs': 2,
'bg': 1
})
return p_model
voxelWidthXY = 36
voxelWidthZ = 24
CAND_THRESHOLD = 150000
outputFile = []
outputFile.append('seriesuid,coordX,coordY,coordZ,probability')
# Keras Parameters
batch_size = 32
num_classes = 2
epochs = 1
input_shape = (1, voxelWidthZ, voxelWidthXY, voxelWidthXY)
K.set_image_data_format('channels_first')
model = Sequential()
model.add(Conv3D(64, kernel_size=(3, 5, 5), activation='relu', input_shape=input_shape))
model.add(MaxPooling3D(pool_size=(3, 3, 3)))
model.add(Dropout(0.20))
model.add(Conv3D(64, (3, 5, 5), activation='relu'))
model.add(MaxPooling3D(pool_size=(2, 2, 2)))
model.add(Dropout(0.20))
model.add(Flatten())
model.add(Dense(300, activation='relu'))
model.add(Dropout(0.20))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=keras.losses.binary_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
# Save the model in order to restart it later
def create_deep_back_projection_network_model_3d(
input_image_size,
number_of_outputs=1,
number_of_base_filters=64,
number_of_feature_filters=256,
number_of_back_projection_stages=7,
convolution_kernel_size=(12, 12, 12),
strides=(8, 8, 8),
last_convolution=(3, 3, 3),
number_of_loss_functions=1):
"""
3-D implementation of the deep back-projection network.
Creates a keras model of the deep back-project network for image super
resolution. More information is provided at the authors' website:
https://www.toyota-ti.ac.jp/Lab/Denshi/iim/members/muhammad.haris/projects/DBPN.html
with the paper available here:
https://arxiv.org/abs/1803.02735
This particular implementation was influenced by the following keras (python)
implementation:
https://github.com/rajatkb/DBPN-Keras
with help from the original author's Caffe and Pytorch implementations:
https://github.com/alterzero/DBPN-caffe
https://github.com/alterzero/DBPN-Pytorch
Arguments
---------
input_image_size : tuple of length 4
Used for specifying the input tensor shape. The shape (or dimension) of
that tensor is the image dimensions followed by the number of channels
(e.g., red, green, and blue).
number_of_outputs : integer
Number of outputs (e.g., 3 for RGB images).
number_of_feature_filters : integer
Number of feature filters.
number_of_base_filters : integer
Number of base filters.
number_of_back_projection_stages : integer
Number of up-down-projection stages.
This number includes the final up block.
convolution_kernel_size : tuple of length 3
Kernel size for certain convolutional layers. The strides are dependent on
the scale factor discussed in original paper. Factors used in the original
implementation are as follows:
2x --> convolution_kernel_size=(6, 6, 6),
4x --> convolution_kernel_size=(8, 8, 8),
8x --> convolution_kernel_size=(12, 12, 12). We default to 8x parameters.
strides : tuple of length 3
Strides for certain convolutional layers. This and the
convolution_kernel_size are dependent on the scale factor discussed in
original paper. Factors used in the original implementation are as follows:
2x --> strides = (2, 2, 2),
4x --> strides = (4, 4, 4),
8x --> strides = (8, 8, 8). We default to 8x parameters.
last_convolution: tuple of length 3
The kernel size for the last convolutional layer.
number_of_loss_functions : integer
The number of data targets, e.g. 2 for 2 targets
Returns
-------
Keras model
A 3-D Keras model defining the network.
Example
-------
>>> model = create_deep_back_projection_network_model_3d((128, 128, 128, 1))
>>> model.summary()
"""
def up_block_3d(L,
number_of_filters=64,
kernel_size=(12, 12, 12),
strides=(8, 8, 8),
include_dense_convolution_layer=True):
if include_dense_convolution_layer == True:
L = Conv3D(filters=number_of_filters,
use_bias=True,
kernel_size=(1, 1, 1),
strides=(1, 1, 1),
padding='same')(L)
L = PReLU(alpha_initializer='zero', shared_axes=[1, 2, 3])(L)
# Scale up
H0 = Conv3DTranspose(filters=number_of_filters,
kernel_size=kernel_size,
strides=strides,
kernel_initializer='glorot_uniform',
padding='same')(L)
H0 = PReLU(alpha_initializer='zero', shared_axes=[1, 2, 3])(H0)
# Scale down
L0 = Conv3D(filters=number_of_filters,
kernel_size=kernel_size,
strides=strides,
kernel_initializer='glorot_uniform',
padding='same')(H0)
L0 = PReLU(alpha_initializer='zero', shared_axes=[1, 2, 3])(L0)
# Residual
E = Subtract()([L0, L])
# Scale residual up
H1 = Conv3DTranspose(filters=number_of_filters,
kernel_size=kernel_size,
strides=strides,
kernel_initializer='glorot_uniform',
padding='same')(E)
H1 = PReLU(alpha_initializer='zero', shared_axes=[1, 2, 3])(H1)
# Output feature map
up_block = Add()([H0, H1])
return (up_block)
def down_block_3d(H,
number_of_filters=64,
kernel_size=(12, 12, 12),
strides=(8, 8, 8),
include_dense_convolution_layer=True):
if include_dense_convolution_layer == True:
H = Conv3D(filters=number_of_filters,
use_bias=True,
kernel_size=(1, 1, 1),
strides=(1, 1, 1),
padding='same')(H)
H = PReLU(alpha_initializer='zero', shared_axes=[1, 2, 3])(H)
# Scale down
L0 = Conv3D(filters=number_of_filters,
kernel_size=kernel_size,
strides=strides,
kernel_initializer='glorot_uniform',
padding='same')(H)
L0 = PReLU(alpha_initializer='zero', shared_axes=[1, 2, 3])(L0)
# Scale up
H0 = Conv3DTranspose(filters=number_of_filters,
kernel_size=kernel_size,
strides=strides,
kernel_initializer='glorot_uniform',
padding='same')(L0)
H0 = PReLU(alpha_initializer='zero', shared_axes=[1, 2, 3])(H0)
# Residual
E = Subtract()([H0, H])
# Scale residual down
L1 = Conv3D(filters=number_of_filters,
kernel_size=kernel_size,
strides=strides,
kernel_initializer='glorot_uniform',
padding='same')(E)
L1 = PReLU(alpha_initializer='zero', shared_axes=[1, 2, 3])(L1)
# Output feature map
down_block = Add()([L0, L1])
return (down_block)
inputs = Input(shape=input_image_size)
# Initial feature extraction
model = Conv3D(filters=number_of_feature_filters,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
kernel_initializer='glorot_uniform')(inputs)
model = PReLU(alpha_initializer='zero', shared_axes=[1, 2, 3])(model)
# Feature smashing
model = Conv3D(filters=number_of_base_filters,
kernel_size=(1, 1, 1),
strides=(1, 1, 1),
padding='same',
kernel_initializer='glorot_uniform')(model)
model = PReLU(alpha_initializer='zero', shared_axes=[1, 2, 3])(model)
# Back projection
up_projection_blocks = []
down_projection_blocks = []
model = up_block_3d(model,
number_of_filters=number_of_base_filters,
kernel_size=convolution_kernel_size,
strides=strides)
up_projection_blocks.append(model)
for i in range(number_of_back_projection_stages):
if i == 0:
model = down_block_3d(model,
number_of_filters=number_of_base_filters,
kernel_size=convolution_kernel_size,
strides=strides)
down_projection_blocks.append(model)
model = up_block_3d(model,
number_of_filters=number_of_base_filters,
kernel_size=convolution_kernel_size,
strides=strides)
up_projection_blocks.append(model)
model = Concatenate()(up_projection_blocks)
else:
model = down_block_3d(model,
number_of_filters=number_of_base_filters,
kernel_size=convolution_kernel_size,
strides=strides,
include_dense_convolution_layer=True)
down_projection_blocks.append(model)
model = Concatenate()(down_projection_blocks)
model = up_block_3d(model,
number_of_filters=number_of_base_filters,
kernel_size=convolution_kernel_size,
strides=strides,
include_dense_convolution_layer=True)
up_projection_blocks.append(model)
model = Concatenate()(up_projection_blocks)
# Final convolution layer
outputs = Conv3D(filters=number_of_outputs,
kernel_size=last_convolution,
strides=(1, 1, 1),
padding='same',
kernel_initializer="glorot_uniform")(model)
if number_of_loss_functions == 1:
deep_back_projection_network_model = Model(inputs=inputs,
outputs=outputs)
else:
outputList = []
for k in range(number_of_loss_functions):
outputList.append(outputs)
deep_back_projection_network_model = Model(inputs=inputs,
outputs=outputList)
return (deep_back_projection_network_model)
def get_3D_Recurrent_DenseUnet():
inputs = Input((img_rows, img_cols, depth, 1))
#list of number of filters per block
depth_cnn = [32, 64, 128, 256]
##start of encoder block
##encoder block1
conv11 = Conv3D(depth_cnn[0], (3, 3, 3), padding='same')(inputs)
conv11 = BatchNormalization()(conv11)
conv11 = Activation('relu')(conv11)
conc11 = concatenate([inputs, conv11], axis=4)
conv12 = Conv3D(depth_cnn[0], (3, 3, 3), padding='same')(conc11)
conv12 = BatchNormalization()(conv12)
conv12 = Activation('relu')(conv12)
conc12 = concatenate([inputs, conv12], axis=4)
perm = Permute((3,1,2,4))(conc12)
pool1 = TimeDistributed(MaxPooling2D((2, 2)))(perm)
pool1 = Permute((2,3,1,4))(pool1)
pool1 = SpatialDropout3D(0.1)(pool1)
#encoder block2
conv21 = Conv3D(depth_cnn[1], (3, 3, 3), padding='same')(pool1)
conv21 = BatchNormalization()(conv21)
conv21 = Activation('relu')(conv21)
conc21 = concatenate([pool1, conv21], axis=4)
conv22 = Conv3D(depth_cnn[1], (3, 3, 3), padding='same')(conc21)
conv22 = BatchNormalization()(conv22)
conv22 = Activation('relu')(conv22)
conc22 = concatenate([pool1, conv22], axis=4)
perm = Permute((3,1,2,4))(conc22)
pool2 = TimeDistributed(MaxPooling2D((2, 2)))(perm)
pool2 = Permute((2,3,1,4))(pool2)
pool2 = SpatialDropout3D(0.1)(pool2)
#encoder block3
conv31 = Conv3D(depth_cnn[2], (3, 3, 3), padding='same')(pool2)
conv31 = BatchNormalization()(conv31)
conv31 = Activation('relu')(conv31)
conc31 = concatenate([pool2, conv31], axis=4)
conv32 = Conv3D(depth_cnn[2], (3, 3, 3), padding='same')(conc31)
conv32 = BatchNormalization()(conv32)
conv32 = Activation('relu')(conv32)
conc32 = concatenate([pool2, conv32], axis=4)
perm = Permute((3,1,2,4))(conc32)
pool3 = TimeDistributed(MaxPooling2D((2, 2)))(perm)
pool3 = SpatialDropout3D(0.1)(pool3)
##end of encoder block
#ConvLSTM block
x = BatchNormalization()(ConvLSTM2D(filters =depth_cnn[3], kernel_size = (3,3), padding='same', return_sequences=True)(pool3))
x = BatchNormalization()(ConvLSTM2D(filters =depth_cnn[3], kernel_size = (3,3), padding='same', return_sequences=True)(x))
x = BatchNormalization()(ConvLSTM2D(filters = depth_cnn[3], kernel_size = (3,3), padding='same', return_sequences=True)(x))
# start of decoder block
# decoder block1
up1 = TimeDistributed(Conv2DTranspose(depth_cnn[2], (2, 2), strides=(2, 2), padding='same'))(x)
up1 = Permute((2,3,1,4))(up1)
up6 = concatenate([up1, conc32], axis=4)
conv61 = Conv3D(depth_cnn[2], (3, 3, 3), padding='same')(up6)
conv61 = BatchNormalization()(conv61)
conv61 = Activation('relu')(conv61)
conc61 = concatenate([up6, conv61], axis=4)
conv62 = Conv3D(depth_cnn[2], (3, 3, 3), padding='same')(conc61)
conv62 = BatchNormalization()(conv62)
conv62 = Activation('relu')(conv62)
conv62 = concatenate([up6, conv62], axis=4)
#decoder block2
up2 = Permute((3,1,2,4))(conv62)
up2 = TimeDistributed(Conv2DTranspose(depth_cnn[1], (2, 2), strides=(2, 2), padding='same'))(up2)
up2 = Permute((2,3,1,4))(up2)
up7 = concatenate([up2, conv22], axis=4)
conv71 = Conv3D(depth_cnn[1], (3, 3, 3), padding='same')(up7)
conv71 = BatchNormalization()(conv71)
conv71 = Activation('relu')(conv71)
conc71 = concatenate([up7, conv71], axis=4)
conv72 = Conv3D(depth_cnn[1], (3, 3, 3), padding='same')(conc71)
conv72 = BatchNormalization()(conv72)
conv72 = Activation('relu')(conv72)
conv72 = concatenate([up7, conv72], axis=4)
#decoder block3
up3 = Permute((3,1,2,4))(conv72)
up3 = TimeDistributed(Conv2DTranspose(depth_cnn[0], (2, 2), strides=(2, 2), padding='same'))(up3)
up3 = Permute((2,3,1,4))(up3)
up8 = concatenate([up3, conv12], axis=4)
conv81 = Conv3D(depth_cnn[0], (3, 3, 3), padding='same')(up8)
conv81 = BatchNormalization()(conv81)
conv81 = Activation('relu')(conv81)
conc81 = concatenate([up8, conv81], axis=4)
conv82 = Conv3D(depth_cnn[0], (3, 3, 3), padding='same')(conc81)
conv82 = BatchNormalization()(conv82)
conv82 = Activation('relu')(conv82)
conc82 = concatenate([up8, conv82], axis=4)
##end of decoder block
conv10 = Conv3D(1, (1, 1, 1), activation='sigmoid')(conc82)
model = Model(inputs=[inputs], outputs=[conv10])
model.compile(optimizer=Adam(lr=1e-4, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.000000199), loss = loss_function, metrics=[dice_coef])
return model
def fn_get_model_convLSTM_tframe_4():
model = Sequential()
model.add(
ConvLSTM2D(filters=64,
kernel_size=(5, 5),
input_shape=(None, width, height, 1),
padding='same',
return_sequences=True,
activation='tanh',
recurrent_activation='hard_sigmoid',
kernel_initializer='glorot_uniform',
unit_forget_bias=True,
dropout=0.3,
recurrent_dropout=0.3))
model.add(BatchNormalization())
model.add(
ConvLSTM2D(filters=32,
kernel_size=(3, 3),
padding='same',
return_sequences=True,
activation='tanh',
recurrent_activation='hard_sigmoid',
kernel_initializer='glorot_uniform',
unit_forget_bias=True,
dropout=0.4,
recurrent_dropout=0.3))
model.add(BatchNormalization())
model.add(
ConvLSTM2D(filters=32,
kernel_size=(3, 3),
padding='same',
return_sequences=True,
activation='tanh',
recurrent_activation='hard_sigmoid',
kernel_initializer='glorot_uniform',
unit_forget_bias=True,
dropout=0.4,
recurrent_dropout=0.3))
model.add(BatchNormalization())
model.add(
ConvLSTM2D(filters=32,
kernel_size=(3, 3),
padding='same',
return_sequences=True,
activation='tanh',
recurrent_activation='hard_sigmoid',
kernel_initializer='glorot_uniform',
unit_forget_bias=True,
dropout=0.4,
recurrent_dropout=0.3))
model.add(BatchNormalization())
model.add(
Conv3D(filters=1,
kernel_size=(1, 1, 1),
activation='sigmoid',
padding='same',
data_format='channels_last'))
### !!! try go_backwards=True !!! ###
# print(model.summary())
return model
def phase_estimator_100k(time_sequence):
seq = Sequential()
seq.add(
ConvLSTM2D(filters=50,
kernel_size=(2, 2),
input_shape=(None, 10, 10, time_sequence),
padding='same',
return_sequences=True))
seq.add(BatchNormalization())
#seq.add(Dropout(0.2))
seq.add(
ConvLSTM2D(filters=75,
kernel_size=(2, 2),
padding='same',
return_sequences=True))
seq.add(BatchNormalization())
#seq.add(Dropout(0.2))
seq.add(
ConvLSTM2D(filters=100,
kernel_size=(2, 2),
padding='same',
return_sequences=True))
seq.add(BatchNormalization())
#seq.add(Dropout(0.1))
seq.add(
ConvLSTM2D(filters=75,
kernel_size=(2, 2),
padding='same',
return_sequences=True))
seq.add(BatchNormalization())
seq.add(
ConvLSTM2D(filters=50,
kernel_size=(2, 2),
padding='same',
return_sequences=True))
seq.add(BatchNormalization())
seq.add(
Conv3D(filters=time_sequence,
kernel_size=(2, 2, 2),
activation='relu',
padding='same',
data_format='channels_last'))
seq.add(BatchNormalization())
seq.add(Dropout(.15))
seq.add(
Conv3D(filters=time_sequence,
kernel_size=(2, 2, 2),
activation='softmax',
padding='same',
data_format='channels_last'))
seq.compile(
loss='categorical_crossentropy',
optimizer='adadelta',
metrics=[metrics.categorical_accuracy, metrics.binary_accuracy])
return seq
segmentation_layer = segmentation_layers[level_number]
if output_layer is None:
output_layer = segmentation_layer
else:
output_layer = Add()([output_layer, segmentation_layer])
if level_number > 0:
output_layer = UpSampling3D(size=(2, 2, 2))(output_layer)
activation_block = Activation(activation = activation_name, name='activation_block')(output_layer)
# survival_block = Activation("linear")(summation_layer)
# activation_block = Dense(1, activation=activation_name, name='activation_block')(output_layer)
# flatten = Flatten(name='flatten')(summation_layer)
# survival_block = Dense(1, activation='linear', name='survival_block')(flatten)
survival_conv_1 = Conv3D(filters=n_level_filters, kernel_size=(3, 3, 3), padding='same', strides=(1, 1, 1), name='survival_conv_1')(summation_layer)
survival_conv_2 = Conv3D(filters=n_level_filters, kernel_size=(3, 3, 3), padding='same', strides=(1, 1, 1), name='survival_conv_2')(survival_conv_1)
dropout = SpatialDropout3D(rate=dropout_rate, data_format='channels_first', name='dropout')(survival_conv_2)
survival_conv_3 = Conv3D(filters=n_level_filters, kernel_size=(3, 3, 3), padding='same', strides=(1, 1, 1), name='survival_conv_3')(dropout)
survival_GAP = GlobalAveragePooling3D(name='survival_GAP')(survival_conv_3)
# flatten = Flatten(name='flatten')(survival_GAP)
# survival_block = Activation("linear", name='survival_block')(flatten)
survival_block = Dense(1, activation='linear', name='survival_block')(survival_GAP)
tumortype_conv_1 = Conv3D(filters=n_level_filters, kernel_size=(3, 3, 3), padding='same', strides=(1, 1, 1), name='tumortype_conv_1')(summation_layer)
tumortype_conv_2 = Conv3D(filters=n_level_filters, kernel_size=(3, 3, 3), padding='same', strides=(1, 1, 1), name='tumortype_conv_2')(tumortype_conv_1)
tumortype_dropout = SpatialDropout3D(rate=dropout_rate, data_format='channels_first', name='tumortype_dropout')(tumortype_conv_2)
tumortype_conv_3 = Conv3D(filters=n_level_filters, kernel_size=(3, 3, 3), padding='same', strides=(1, 1, 1), name='tumortype_conv_3')(tumortype_dropout)
tumortype_GAP = GlobalAveragePooling3D(name='tumortype_GAP')(tumortype_conv_3)
# flatten = Flatten(name='flatten')(tumortype_GAP)
# tumortype_block = Activation("linear", name='tumortype_block')(flatten)
# x reshape, x dimension is set to 5 (sample,dim1,dim2,dim3,channel)
print("shape of x_train before: ", x_train.shape)
x_train = np.expand_dims(x_train, axis=4)
print("shape of x_train after: ", x_train.shape)
# y reshape, y dimension is set to 5 (sample,dim1,dim2,dim3,channel)
print("shape of y_train before: ", y_train.shape)
#y_train = np.expand_dims(y_train, axis=4)
#print("shape of x_train after: ",y_train.shape)
############# the sequantial model ##############################################################################
model = Sequential()
model.add(
Conv3D(10,
kernel_size=(3, 3, 3),
activation='relu',
input_shape=(9, 9, 9, 1),
padding='same'))
model.add(Conv3D(12, kernel_size=(3, 3, 3), activation='relu', padding='same'))
model.add(Conv3D(24, kernel_size=(3, 3, 3), activation='relu',
padding='valid'))
model.add(Conv3D(48, kernel_size=(3, 3, 3), activation='relu', padding='same'))
model.add(Conv3D(72, kernel_size=(3, 3, 3), activation='relu',
padding='valid'))
model.add(Conv3D(96, kernel_size=(3, 3, 3), activation='relu',
padding='valid'))
model.add(Conv3D(96, kernel_size=(3, 3, 3), activation='relu',
padding='valid'))
#model.add(MaxPooling3D(pool_size=(3,3,3)))
model.add(Flatten())
#model.add(Dense(300,activation='relu'))
def build_model2():
'''
保证输入为None 60 132 输出为None 30 162
:return:
'''
from keras.layers.advanced_activations import LeakyReLU
model = Sequential()
#'___________________________________________________'
model.add(
ConvLSTM2D(
# 可能这个filters应该再多一些
filters=5,
# 每一次只有一个kernel，一次卷积出100个结果，卷积的结果之间会有LSTM联系
# 注意用summary检查参数数量，参数太多不是很好，这里如果filter是100，参数会到达1000 0000的数量级，所以这里filter改成10
input_shape=(60, 132, 1, 1), #(n_frame, width, height, channel)
kernel_size=(132, 1),
padding="same",
return_sequences=True,
#activation="selu"
))
model.add(LeakyReLU())
model.add(BatchNormalization())
#model.add(Dropout(0.75))
#'___________________________________________________'
model.add(
ConvLSTM2D(
filters=10,
kernel_size=(132, 1),
padding="same",
return_sequences=True,
#activation="selu"
))
model.add(LeakyReLU())
model.add(BatchNormalization())
model.add(Dropout(0.3))
#'___________________________________________________'
model.add(
ConvLSTM2D(
filters=10,
kernel_size=(6, 1),
padding="same",
return_sequences=True,
#activation="selu"
))
model.add(BatchNormalization())
model.add(Dropout(0.3))
#'___________________________________________________'
model.add(
Conv3D(
filters=5,
kernel_size=(132, 1, 2),
strides=(2, 1, 1), # 这里用步长压缩shape为(30,132,1,1)
#activation="selu",
padding="same",
))
model.add(LeakyReLU())
model.add(BatchNormalization())
model.add(Dropout(0.3))
#'___________________________________________________'
model.add(
Conv3D(
filters=10,
kernel_size=(132, 1, 3),
strides=(1, 1, 1),
#activation="selu",
padding="same",
data_format="channels_last"))
model.add(LeakyReLU())
model.add(BatchNormalization())
model.add(Dropout(0.3))
#'___________________________________________________'
model.add(
Conv3D(
filters=1,
kernel_size=(132, 1, 5),
strides=(1, 1, 1),
#activation="tanh",
padding="same",
data_format="channels_last"))
model.add(LeakyReLU())
return model
def main():
parser = argparse.ArgumentParser(
description='simple 3D convolution for action recognition')
parser.add_argument('--batch', type=int, default=128)
parser.add_argument('--epoch', type=int, default=100)
parser.add_argument('--videos', type=str, default='UCF101',
help='directory where videos are stored')
parser.add_argument('--nclass', type=int, default=101)
parser.add_argument('--output', type=str, required=True)
parser.add_argument('--color', type=bool, default=False)
parser.add_argument('--skip', type=bool, default=True)
parser.add_argument('--depth', type=int, default=10)
args = parser.parse_args()
img_rows, img_cols, frames = 32, 32, args.depth
channel = 3 if args.color else 1
fname_npz = 'dataset_{}_{}_{}.npz'.format(
args.nclass, args.depth, args.skip)
vid3d = videoto3d1.Videoto3D(img_rows, img_cols, frames)
nb_classes = args.nclass
if os.path.exists(fname_npz):
loadeddata = np.load(fname_npz)
X, Y = loadeddata["X"], loadeddata["Y"]
else:
x, y = loaddata(args.videos, vid3d, args.nclass,
args.output, args.color, args.skip)
X = x.reshape((x.shape[0], img_rows, img_cols, frames, channel))
Y = np_utils.to_categorical(y, nb_classes)
X = X.astype('float32')
#np.savez(fname_npz, X=X, Y=Y)
#print('Saved dataset to dataset.npz.')
print('X_shape:{}\nY_shape:{}'.format(X.shape, Y.shape))
# Define model
model = Sequential()
model.add(Conv3D(32, kernel_size=(3, 3, 3), input_shape=(
X.shape[1:]), padding="same"))
model.add(Activation('relu'))
model.add(Conv3D(32, padding="same", kernel_size=(3, 3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling3D(pool_size=(3, 3, 3), padding="same"))
model.add(Dropout(0.25))
model.add(Conv3D(64, padding="same", kernel_size=(3, 3, 3)))
model.add(Activation('relu'))
model.add(Conv3D(64, padding="same", kernel_size=(3, 3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling3D(pool_size=(3, 3, 3), padding="same"))
model.add(Dropout(0.25))
model.add(Conv3D(64, padding="same", kernel_size=(3, 3, 3)))
model.add(Activation('relu'))
model.add(Conv3D(64, padding="same", kernel_size=(3, 3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling3D(pool_size=(3, 3, 3), padding="same"))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(nb_classes, activation='softmax'))
model.compile(loss=categorical_crossentropy,
optimizer='rmsprop', metrics=['accuracy'])
model.summary()
plot_model(model, show_shapes=True,
to_file=os.path.join(args.output, 'model.png'))
X_train, X_test, Y_train, Y_test = train_test_split(
X, Y, test_size=0.2, random_state=43)
history = model.fit(X_train, Y_train, validation_data=(X_test, Y_test), batch_size=args.batch,
epochs=args.epoch, verbose=1, shuffle=True)
model.evaluate(X_test, Y_test, verbose=0)
model_json = model.to_json()
if not os.path.isdir(args.output):
os.makedirs(args.output)
with open(os.path.join(args.output, 'ucf101_3dcnnmodel.json'), 'w') as json_file:
json_file.write(model_json)
model.save_weights(os.path.join(args.output, 'ucf101_3dcnnmodel.hd5'))
loss, acc = model.evaluate(X_test, Y_test, verbose=0)
print('Test loss:', loss)
print('Test accuracy:', acc)
plot_history(history, args.output)
save_history(history, args.output)
def CLoST3D(city,
X_train,
y_train,
conv_filt=64,
kernel_sz=(2, 3, 3),
mask=np.empty(0),
lstm=None,
lstm_number=0,
add_external_info=False):
# Input:
# - mask: np.array. Filter that is applied to the data output to the model. If not passed, no filter is applied
# - lstm: int. Parameter to pass to the LSTM layer. If equal to None, the LSTM layer is not added.
# - add_external_info: bool. Parameter to insert external information or not.
X_train, ext_train = X_train # split flow volumes and ext features
main_inputs = []
start = Input(shape=(X_train.shape[1], X_train.shape[2], X_train.shape[3],
2))
main_inputs.append(start)
main_output = main_inputs[0]
x = Conv3D(conv_filt / 2, kernel_size=kernel_sz,
activation='relu')(main_output)
x = MaxPooling3D(pool_size=(1, 2, 2))(x)
x = Dropout(0.25)(x)
x = Conv3D(conv_filt,
kernel_size=kernel_sz,
activation='relu',
padding='same')(x)
x = MaxPooling3D(pool_size=(1, 2, 2))(x)
if city == 'BJ':
x = Dropout(0.25)(x)
x = Conv3D(conv_filt,
kernel_size=kernel_sz,
activation='relu',
padding='same')(x)
x = MaxPooling3D(pool_size=(1, 2, 2))(x)
x = Flatten()(x)
x = Dense(128, activation='relu')(x)
if lstm != None:
x = Reshape((x.shape[1], 1))(x)
for num in range(lstm_number):
if city == 'BJ':
x = LSTM(int(lstm / (num + 1)), return_sequences=True)(x)
elif city == 'NY':
x = LSTM(int(lstm), return_sequences=True)(x)
x = Flatten()(x)
if add_external_info:
external_input = Input(shape=ext_train.shape[1:])
main_inputs.append(external_input)
x_ext = Dense(units=10, activation='relu')(external_input)
x_ext = Dense(units=reduce(lambda e1, e2: e1 * e2, y_train.shape[1:]),
activation='relu')(x_ext)
# x = Flatten()(x)
x = Concatenate(axis=-1)([x, x_ext])
x = Dense(reduce(lambda e1, e2: e1 * e2, y_train.shape[1:]))(x)
x = Reshape(y_train.shape[1:])(x)
x = Activation(swish)(x)
if mask.shape[0] != 0:
x = Lambda(lambda el: el * mask)(x)
model = Model(main_inputs, x)
return model
def discriminator_block_conditionnal(self, name):
"""Creates a discriminator model that takes an image as input and outputs a single value, representing whether
the input is real or generated. Unlike normal GANs, the output is not sigmoid and does not represent a
probability!
Instead, the output should be as large and negative as possible for generated inputs and as large and positive
as possible for real inputs.
Note that the improved WGAN paper suggests that BatchNormalization should not be used in the discriminator."""
# In:
im = Input(shape=(2, self.image_row, self.image_column,
self.image_depth),
name='dis_input')
res = Input(shape=(1, self.image_row, self.image_column,
self.image_depth),
name='dis_input_res')
inputs = Concatenate(axis=-4)([im, res])
# Input 64
disnet = Conv3D(self.discriminator_kernel * 1,
4,
strides=2,
padding='same',
kernel_initializer='he_normal',
data_format='channels_first',
name=name + '_conv_dis_1')(inputs)
disnet = LeakyReLU(0.01)(disnet)
# Hidden 1 : 32
disnet = Conv3D(self.discriminator_kernel * 2,
4,
strides=2,
padding='same',
kernel_initializer='he_normal',
data_format='channels_first',
name=name + '_conv_dis_2')(disnet)
disnet = LeakyReLU(0.01)(disnet)
# Hidden 2 : 16
disnet = Conv3D(self.discriminator_kernel * 4,
4,
strides=2,
padding='same',
kernel_initializer='he_normal',
data_format='channels_first',
name=name + '_conv_dis_3')(disnet)
disnet = LeakyReLU(0.01)(disnet)
# Hidden 3 : 8
disnet = Conv3D(self.discriminator_kernel * 8,
4,
strides=2,
padding='same',
kernel_initializer='he_normal',
data_format='channels_first',
name=name + '_conv_dis_4')(disnet)
disnet = LeakyReLU(0.01)(disnet)
# Hidden 4 : 4
disnet = Conv3D(self.discriminator_kernel * 16,
4,
strides=2,
padding='same',
kernel_initializer='he_normal',
data_format='channels_first',
name=name + '_conv_dis_5')(disnet)
disnet = LeakyReLU(0.01)(disnet)
# Decision : 2
decision = Conv3D(1,
2,
strides=1,
use_bias=False,
kernel_initializer='he_normal',
data_format='channels_first',
name='dis_decision')(disnet)
decision = Reshape((1, ))(decision)
model = Model(inputs=[im, res], outputs=[decision], name=name)
return model
# Metrics
def dice(y_true, y_pred):
y_true_f = K.flatten(y_true)
y_pred_f = K.flatten(y_pred)
intersection = K.sum(y_true_f * y_pred_f)
return (2. * intersection + 1) / (K.sum(y_true_f) + K.sum(y_pred_f) + 1)
# Losses
def dice_loss(y_true, y_pred):
return 1-dice(y_true, y_pred)
# Model
K.clear_session()
I1 = Input(shape=(64, 64, 64, 4))
conv1 = Conv3D(32, (3, 3, 3), activation='relu', padding='same')(I1)
conv1 = Conv3D(32, (3, 3, 3), activation='relu', padding='same')(conv1)
pool1 = MaxPool3D(pool_size=(2, 2, 2))(conv1)
d1 = Dropout(0.1) (pool1)
conv2 = Conv3D(64, (3, 3, 3), activation='relu', padding='same')(d1)
conv2 = Conv3D(64, (3, 3, 3), activation='relu', padding='same')(conv2)
pool2 = MaxPool3D(pool_size=(2, 2, 2))(conv2)
d2 = Dropout(0.1) (pool2)
conv3 = Conv3D(128, (3, 3, 3), activation='relu', padding='same')(d2)
conv3 = Conv3D(128, (3, 3, 3), activation='relu', padding='same')(conv3)
pool3 = MaxPool3D(pool_size=(2, 2, 2))(conv3)
d3 = Dropout(0.2) (pool3)
conv4 = Conv3D(256, (3, 3, 3), activation='relu', padding='same')(d3)
conv4 = Conv3D(256, (3, 3, 3), activation='relu', padding='same')(conv4)
pool4 = MaxPool3D(pool_size=(2, 2, 2))(conv4)
d4 = Dropout(0.2) (pool4)
def generator_block_u_net(
self,
name): # generateur meme dim en entree et sortie si multiple de 4
#
inputs = Input(shape=(1, self.image_row, self.image_column,
self.image_depth))
# Representation
gennet = ReflectPadding3D(padding=3)(inputs)
gennet = Conv3D(self.generator_kernel,
7,
strides=1,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_conv1',
data_format='channels_first')(gennet)
gennet = InstanceNormalization3D(name=name +
'_gen_isnorm_conv1')(gennet)
gennet = Activation('relu')(gennet)
# resblock :
gennet = resnet_blocks(gennet,
self.generator_kernel,
name=name + '_gen_block')
# Downsampling 1
gennet_down_1 = ReflectPadding3D(padding=1)(gennet)
gennet_down_1 = Conv3D(self.generator_kernel * 2,
3,
strides=2,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_conv2',
data_format='channels_first')(gennet_down_1)
gennet_down_1 = InstanceNormalization3D(
name=name + '_gen_isnorm_conv2')(gennet_down_1)
gennet_down_1 = Activation('relu')(gennet_down_1)
# resblock 1 :
gennet_down_1 = resnet_blocks(gennet_down_1,
self.generator_kernel * 2,
name=name + '_gen_block1')
# Downsampling 2
gennet_down_2 = ReflectPadding3D(padding=1)(gennet_down_1)
gennet_down_2 = Conv3D(self.generator_kernel * 4,
3,
strides=2,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_conv3',
data_format='channels_first')(gennet_down_2)
gennet_down_2 = InstanceNormalization3D(
name=name + '_gen_isnorm_conv3')(gennet_down_2)
gennet_down_2 = Activation('relu')(gennet_down_2)
# resblock 2
gennet_down_2 = resnet_blocks(gennet_down_2,
self.generator_kernel * 4,
name=name + '_gen_block2')
# Upsampling X2 down_2 :
gennet_up_1 = UpSampling3D(size=(2, 2, 2),
data_format='channels_first')(gennet_down_2)
gennet_up_1 = ReflectPadding3D(padding=1)(gennet_up_1)
gennet_up_1 = Conv3D(self.generator_kernel * 2,
3,
strides=1,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_deconv1',
data_format='channels_first')(gennet_up_1)
gennet_up_1 = InstanceNormalization3D(
name=name + '_gen_isnorm_deconv1')(gennet_up_1)
gennet_up_1 = Activation('relu')(gennet_up_1)
# del gennet_down_2
# Concatenante gennet_up_1 with gennet_down_1
gennet_concate_1 = Concatenate(axis=-4)([gennet_up_1, gennet_down_1])
# del gennet_up_1
# del gennet_down_1
# Upsampling 2
gennet_up_2 = UpSampling3D(
size=(2, 2, 2), data_format='channels_first')(gennet_concate_1)
gennet_up_2 = ReflectPadding3D(padding=1)(gennet_up_2)
gennet_up_2 = Conv3D(self.generator_kernel,
3,
strides=1,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_deconv2',
data_format='channels_first')(gennet_up_2)
gennet_up_2 = InstanceNormalization3D(
name=name + '_gen_isnorm_deconv2')(gennet_up_2)
gennet_up_2 = Activation('relu')(gennet_up_2)
# Concatenante gennet_up_2 with gennet_down_1
gennet_concate_2 = Concatenate(axis=-4)([gennet_up_2, gennet])
# del gennet_concate_1
# del gennet_up_2
# Reconstruction
gennet_concate_2 = ReflectPadding3D(padding=3)(gennet_concate_2)
gennet_concate_2 = Conv3D(
2,
7,
strides=1,
kernel_initializer=gen_initializer,
use_bias=False,
name=name + '_gen_1conv',
data_format='channels_first')(gennet_concate_2)
predictions = gennet_concate_2
predictions = activation_SegSRGAN(is_residual=self.is_residual)(
[predictions, inputs])
model = Model(inputs=inputs, outputs=predictions, name=name)
return model
def build_model(input_shape=(4, 160, 192, 128),
output_channels=3,
weight_L2=0.1,
weight_KL=0.1,
dice_e=1e-8,
input_dropout_rate=0.5):
"""
build_model_ae(input_shape=(4, 160, 192, 128), output_channels=3, weight_L2=0.1, weight_KL=0.1)
-------------------------------------------
Creates the model used in the BRATS2018 winning solution
by Myronenko A. (https://arxiv.org/pdf/1810.11654.pdf)
Parameters
----------
`input_shape`: A 4-tuple, optional.
Shape of the input image. Must be a 4D image of shape (c, H, W, D),
where, each of H, W and D are divisible by 2^4, and c is divisible by 4.
Defaults to the crop size used in the paper, i.e., (4, 160, 192, 128).
`output_channels`: An integer, optional.
The no. of channels in the output. Defaults to 3 (BraTS 2018 format).
`weight_L2`: A real number, optional
The weight to be given to the L2 loss term in the loss function. Adjust to get best
results for your task. Defaults to 0.1.
`weight_KL`: A real number, optional
The weight to be given to the KL loss term in the loss function. Adjust to get best
results for your task. Defaults to 0.1.
`dice_e`: Float, optional
A small epsilon term to add in the denominator of dice loss to avoid dividing by
zero and possible gradient explosion. This argument will be passed to loss_gt function.
'input_dropout' : Float, optiional
Dropout to be used on the input layer. Use '0' for no dropout (pass through all inputs)
Returns
-------
`model`: A keras.models.Model instance
The created model.
"""
c, H, W, D = input_shape
assert len(input_shape) == 4, "Input shape must be a 4-tuple"
assert (c % 4) == 0, "The no. of channels must be divisible by 4"
assert (H % 16) == 0 and (W % 16) == 0 and (D % 16) == 0, \
"All the input dimensions must be divisible by 16"
# -------------------------------------------------------------------------
# Encoder
# -------------------------------------------------------------------------
## Input Layer
inp = Input(input_shape)
# ## Dropout Input Layer (starting 50% for removal of 2 modalities)
# x = SpatialDropout3D(input_dropout_rate, data_format='channels_first')(x)
## The Initial Block
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_x1')(inp)
## Dropout (0.2)
x = SpatialDropout3D(0.2, data_format='channels_first')(x)
## Green Block x1 (output filters = 32)
x1 = green_block(x, 32, name='x1')
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_32')(x1)
## Green Block x2 (output filters = 64)
x = green_block(x, 64, name='Enc_64_1')
x2 = green_block(x, 64, name='x2')
x = Conv3D(filters=64,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_64')(x2)
## Green Blocks x2 (output filters = 128)
x = green_block(x, 128, name='Enc_128_1')
x3 = green_block(x, 128, name='x3')
x = Conv3D(filters=128,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Enc_DownSample_128')(x3)
## Green Blocks x4 (output filters = 256)
x = green_block(x, 256, name='Enc_256_1')
x = green_block(x, 256, name='Enc_256_2')
x = green_block(x, 256, name='Enc_256_3')
x4 = green_block(x, 256, name='x4')
# -------------------------------------------------------------------------
# Decoder
# -------------------------------------------------------------------------
## GT (Groud Truth) Part
# -------------------------------------------------------------------------
### Green Block x1 (output filters=128)
x = Conv3D(filters=128,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_128')(x4)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_128')(x)
x = Add(name='Input_Dec_GT_128')([x, x3])
x = green_block(x, 128, name='Dec_GT_128')
### Green Block x1 (output filters=64)
x = Conv3D(filters=64,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_64')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_64')(x)
x = Add(name='Input_Dec_GT_64')([x, x2])
x = green_block(x, 64, name='Dec_GT_64')
### Green Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_GT_ReduceDepth_32')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_GT_UpSample_32')(x)
x = Add(name='Input_Dec_GT_32')([x, x1])
x = green_block(x, 32, name='Dec_GT_32')
### Blue Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_Dec_GT_Output')(x)
### Output Block
out_GT = Conv3D(
filters=output_channels, # No. of tumor classes is 3
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
activation='sigmoid',
name='Dec_GT_Output')(x)
## VAE (Variational Auto Encoder) Part
# -------------------------------------------------------------------------
### VD Block (Reducing dimensionality of the data)
x = GroupNormalization(groups=8, axis=1, name='Dec_VAE_VD_GN')(x4)
x = Activation('relu', name='Dec_VAE_VD_relu')(x)
x = Conv3D(filters=16,
kernel_size=(3, 3, 3),
strides=2,
padding='same',
data_format='channels_first',
name='Dec_VAE_VD_Conv3D')(x)
# Not mentioned in the paper, but the author used a Flattening layer here.
x = Flatten(name='Dec_VAE_VD_Flatten')(x)
x = Dense(256, name='Dec_VAE_VD_Dense')(x)
### VDraw Block (Sampling)
z_mean = Dense(128, name='Dec_VAE_VDraw_Mean')(x)
z_var = Dense(128, name='Dec_VAE_VDraw_Var')(x)
x = Lambda(sampling, name='Dec_VAE_VDraw_Sampling')([z_mean, z_var])
### VU Block (Upsizing back to a depth of 256)
x = Dense((c // 4) * (H // 16) * (W // 16) * (D // 16))(x)
x = Activation('relu')(x)
x = Reshape(((c // 4), (H // 16), (W // 16), (D // 16)))(x)
x = Conv3D(filters=256,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_256')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_256')(x)
### Green Block x1 (output filters=128)
x = Conv3D(filters=128,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_128')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_128')(x)
x = green_block(x, 128, name='Dec_VAE_128')
### Green Block x1 (output filters=64)
x = Conv3D(filters=64,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_64')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_64')(x)
x = green_block(x, 64, name='Dec_VAE_64')
### Green Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_ReduceDepth_32')(x)
x = UpSampling3D(size=2,
data_format='channels_first',
name='Dec_VAE_UpSample_32')(x)
x = green_block(x, 32, name='Dec_VAE_32')
### Blue Block x1 (output filters=32)
x = Conv3D(filters=32,
kernel_size=(3, 3, 3),
strides=1,
padding='same',
data_format='channels_first',
name='Input_Dec_VAE_Output')(x)
### Output Block
out_VAE = Conv3D(filters=4,
kernel_size=(1, 1, 1),
strides=1,
data_format='channels_first',
name='Dec_VAE_Output')(x)
# Build and Compile the model
out = out_GT
model = Model(inp, outputs=[out, out_VAE]) # Create the model
model.compile(adam(lr=1e-5), [
loss_gt(dice_e),
loss_VAE(input_shape,
z_mean,
z_var,
weight_L2=weight_L2,
weight_KL=weight_KL)
],
metrics=[dice_coefficient])
return model
