x_test = x_test[:-(len(x_test) % batch_size)]
y_train = f['y_train'].value[:]
y_train = y_train[:-(len(y_train) % batch_size)]
y_val = f['y_val'].value[:]
y_val = y_val[:-(len(y_val) % batch_size)]
y_test = f['y_test'].value[:]
y_test = y_test[:-(len(y_test) % batch_size)]
print(y_train.shape, y_test.shape)
model = Sequential()
#Custom model for loss testing
model.add(
Conv3D(16,
input_shape=(20, 20, 20, 12),
data_format='channels_last',
kernel_size=(3, 3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling3D(pool_size=(3, 3, 3), strides=(2, 2, 2)))
model.add(Conv3D(32, kernel_size=(3, 3, 3)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling3D(pool_size=(3, 3, 3), strides=(2, 2, 2)))
model.add(Conv3D(32, kernel_size=(1, 1, 1)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv3D(64, kernel_size=(1, 1, 1)))
model.add(BatchNormalization())
def C3D_model(weights_path=None,
summary=False,
trainable=True,
num_layers_remove=0):
# 1st layer group
input1 = Input((16, 112, 112, 3))
op1 = (Conv3D(64, (3, 3, 3),
activation="relu",
name="conv1",
input_shape=(16, 112, 112, 3),
strides=(1, 1, 1),
padding="same"))(input1)
pool1 = (MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
name="pool1",
padding="valid"))(op1)
# 2nd layer group
op2 = (Conv3D(128, (3, 3, 3),
activation="relu",
name="conv2",
strides=(1, 1, 1),
padding="same"))(pool1)
pool2 = (MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
name="pool2",
padding="valid"))(op2)
# 3rd layer group
op3 = (Conv3D(256, (3, 3, 3),
activation="relu",
name="conv3a",
strides=(1, 1, 1),
padding="same"))(pool2)
op4 = (Conv3D(256, (3, 3, 3),
activation="relu",
name="conv3b",
strides=(1, 1, 1),
padding="same"))(op3)
pool3 = (MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
name="pool3",
padding="valid"))(op4)
# 4th layer group
op5 = (Conv3D(512, (3, 3, 3),
activation="relu",
name="conv4a",
strides=(1, 1, 1),
padding="same"))(pool3)
op6 = (Conv3D(512, (3, 3, 3),
activation="relu",
name="conv4b",
strides=(1, 1, 1),
padding="same"))(op5)
pool4 = (MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
name="pool4",
padding="valid"))(op6)
# 5th layer group
op7 = (Conv3D(512, (3, 3, 3),
activation="relu",
name="conv5a",
strides=(1, 1, 1),
padding="same"))(pool4)
op8 = (Conv3D(512, (3, 3, 3),
activation="relu",
name="conv5b",
strides=(1, 1, 1),
padding="same"))(op7)
pad1 = (ZeroPadding3D(padding=(0, 1, 1)))(op8)
pool5 = (MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
name="pool5",
padding="valid"))(pad1)
flat = (Flatten())(pool5)
# FC layers group
dense1 = (Dense(4096, activation='relu', name='fc6'))(flat)
drop1 = (Dropout(.5))(dense1)
dense2 = (Dense(4096, activation='relu', name='fc7'))(drop1)
drop2 = (Dropout(.5))(dense2)
dense3 = (Dense(487, activation='softmax', name='fc8'))(drop2)
model1 = tf.keras.Model(inputs=input1, outputs=[dense3, op8])
model2 = tf.keras.Model(inputs=input1, outputs=dense3)
model3 = tf.keras.Model(inputs=input1, outputs=op8)
model4 = tf.keras.Model(inputs=input1, outputs=[dense3, op6])
model5 = tf.keras.Model(inputs=input1, outputs=op6)
model6 = tf.keras.Model(inputs=input1, outputs=[dense3, op4])
model7 = tf.keras.Model(inputs=input1, outputs=op4)
return model1, model2, model3, model4, model5, model6, model7
def __init__(self):
# 不指定inputshape仍然work
super(visionModel, self).__init__()
self.c1 = Conv3D(filters=64,
kernel_size=(2, 5, 5),
strides=(1, 1, 1),
input_shape=(frameLen, 72, 128, 3),
padding='same') # 卷积层
# self.c1 = Conv2D(filters=64, kernel_size=(3, 3), padding='same', strides=1) # 卷积层
self.b1 = BatchNormalization()
self.a1 = Activation('relu')
self.c2 = Conv3D(filters=64,
kernel_size=(2, 5, 5),
strides=(1, 1, 1),
padding='same') # 卷积层
# self.c2 = Conv2D(filters=64, kernel_size=(3, 3), strides=1, padding='same') # 卷积层
self.b2 = BatchNormalization()
self.a2 = Activation('relu')
self.p1 = MaxPool3D(pool_size=(2, 3, 3),
strides=(2, 2, 2),
padding='same') # 池化层
# self.p1 = MaxPool2D(pool_size=(2, 2), strides=2, padding='same') # 池化层
self.d1 = Dropout(0.2)
self.c3 = Conv3D(filters=128, kernel_size=(2, 5, 5),
padding='same') # 卷积层
# self.c3 = Conv2D(filters=128, kernel_size=(3, 3), padding='same', strides=1) # 卷积层
self.b3 = BatchNormalization()
self.a3 = Activation('relu')
self.c4 = Conv3D(filters=128, kernel_size=(2, 5, 5),
padding='same') # 卷积层
self.b4 = BatchNormalization()
self.a4 = Activation('relu')
self.p2 = MaxPool3D(pool_size=(2, 3, 3),
strides=(2, 2, 2),
padding='same') # 池化层
self.d2 = Dropout(0.2)
self.c5 = Conv3D(filters=256, kernel_size=(2, 5, 5),
padding='same') # 卷积层
# self.c5 = Conv2D(filters=256, kernel_size=(3, 3), padding='same', strides=1) # 卷积层
self.b5 = BatchNormalization()
self.a5 = Activation('relu')
# 先搞5层试试
self.c6 = Conv2D(filters=256,
kernel_size=(3, 3),
padding='same',
strides=1) # 卷积层
self.b6 = BatchNormalization()
self.a6 = Activation('relu')
self.c7 = Conv2D(filters=256,
kernel_size=(3, 3),
padding='same',
strides=1) # 卷积层
self.b7 = BatchNormalization()
self.a7 = Activation('relu')
self.p3 = MaxPool2D(pool_size=(2, 2), strides=2, padding='same') # 池化层
self.d3 = Dropout(0.2)
self.c8 = Conv2D(filters=512,
kernel_size=(3, 3),
padding='same',
strides=1) # 卷积层
self.b8 = BatchNormalization()
self.a8 = Activation('relu')
self.c9 = Conv2D(filters=512,
kernel_size=(3, 3),
padding='same',
strides=1) # 卷积层
self.b9 = BatchNormalization()
self.a9 = Activation('relu')
self.c10 = Conv2D(filters=512,
kernel_size=(3, 3),
padding='same',
strides=1) # 卷积层
self.b10 = BatchNormalization()
self.a10 = Activation('relu')
self.p4 = MaxPool2D(pool_size=(2, 2), strides=2, padding='same') # 池化层
self.d4 = Dropout(0.2)
self.c11 = Conv2D(filters=512,
kernel_size=(3, 3),
padding='same',
strides=1) # 卷积层
self.b11 = BatchNormalization()
self.a11 = Activation('relu')
self.c12 = Conv2D(filters=512,
kernel_size=(3, 3),
padding='same',
strides=1) # 卷积层
self.b12 = BatchNormalization()
self.a12 = Activation('relu')
self.c13 = Conv2D(filters=512,
kernel_size=(3, 3),
padding='same',
strides=1) # 卷积层
self.b13 = BatchNormalization()
self.a13 = Activation('relu')
self.p5 = MaxPool2D(pool_size=(2, 2), strides=2, padding='same') # 池化层
self.d5 = Dropout(0.2)
self.gru1 = GRU(1024,
return_sequences=True,
stateful=True,
recurrent_initializer='glorot_uniform')
self.layerNor1 = LayerNormalization()
self.drop1 = Dropout(0.2)
self.gru2 = GRU(512,
return_sequences=False,
stateful=True,
recurrent_initializer='glorot_uniform')
self.dense1 = Dense(256, activation='relu')
self.dense2 = Dense(1, activation='linear')
self.flatten = Flatten()
self.f1 = Dense(512, activation='relu')
self.d1 = Dropout(0.2)
self.f2 = Dense(512, activation='relu')
self.d2 = Dropout(0.2)
self.f3 = Dense(10, activation='softmax')
def asymmetric_3D_network(image_shape,
numInitChannels=16,
fixed_dropout=0.0,
optimizer="sgd",
lr=0.001,
t_downsmp_layer=4):
"""Create the assymetric network proposed in Cheng et al.
Parameters
----------
image_shape : array of 3 int
Dimensions of the input image.
numInitChannels : int, optional
Number of convolution channels to start with. In each
downsampling/upsampling the number of filters are multiplied/divided
by ``2``.
fixed_dropout : float, optional
Dropout value to be fixed. If no value is provided the default
behaviour will be to select a piramidal value stating from 0.1 and
reaching 0.3 value.
optimizer : str, optional
Optimizer used to minimize the loss function. Posible options: ``sgd``
or ``adam``.
lr : float, optional
Learning rate value.
t_downsmp_layer : int, optional
Degree of randomness in the sampling pattern which corresponds to the
``t`` value defined in the paper for the proposed stochastic
downsampling layer.
Returns
-------
model : Keras model
Asymmetric network proposed in Cheng et al. model.
Here is a picture of the network extracted from the original paper:
.. image:: img/cheng_network.png
:width: 90%
:align: center
"""
inputs = Input(image_shape)
# Input block
channels = numInitChannels
c1 = Conv3D(channels, (3, 3, 3),
activation=None,
strides=(2, 2, 2),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01))(inputs)
m1 = MaxPooling3D((2, 2, 2))(inputs)
x = concatenate([c1, m1])
# First encode block sequence
for i in range(2):
channels += 8
x = encode_block(x,
channels,
t_downsmp_layer=t_downsmp_layer,
fixed_dropout=fixed_dropout)
# 1st downsample block
channels += 8
x = encode_block(x,
channels,
downsample=True,
t_downsmp_layer=t_downsmp_layer,
fixed_dropout=fixed_dropout)
# Second encode block sequence
for i in range(3):
channels += 8
x = encode_block(x,
channels,
t_downsmp_layer=t_downsmp_layer,
fixed_dropout=fixed_dropout)
# 2nd downsample block
channels += 8
x = encode_block(x,
channels,
downsample=True,
t_downsmp_layer=t_downsmp_layer,
fixed_dropout=fixed_dropout)
# Third encode block sequence
for i in range(6):
channels += 8
x = encode_block(x,
channels,
t_downsmp_layer=t_downsmp_layer,
fixed_dropout=fixed_dropout)
# 1st upsample block
channels = 64
x = decode_block(x, channels, upsample=True)
# First decode block sequence
for i in range(4):
x = decode_block(x, channels)
# 2nd upsample block
channels = int(channels / 2)
x = decode_block(x, channels, upsample=True)
# Second decode block sequence
for i in range(2):
x = decode_block(x, channels)
# Last transpose conv
outputs = Conv3DTranspose(2, (2, 2, 2),
activation="softmax",
strides=(2, 2, 2),
kernel_regularizer=l2(0.01))(x)
model = Model(inputs=[inputs], outputs=[outputs])
if optimizer == "sgd":
opt = tf.keras.optimizers.SGD(lr=lr,
momentum=0.90,
decay=0.0,
nesterov=False)
elif optimizer == "adam":
opt = tf.keras.optimizers.Adam(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
else:
raise ValueError("Error: optimizer value must be 'sgd' or 'adam'")
model.compile(optimizer=opt,
loss=jaccard_loss_cheng2017,
metrics=[jaccard_index_softmax])
return model
def CreateModel(feats,
units=[16, 8, 4, 8, 16],
dropout=.25,
batch_norm=True,
filt_size=3,
pool_size=2):
print('Creating ' + feats.model_type + ' Model')
print('Input shape: ' + str(feats.input_shape))
nunits = len(units)
##---LSTM - Many to two, sequence of time to classes
#Units must be at least two
if feats.model_type == 'LSTM':
if nunits < 2:
print('Warning: Need at least two layers for LSTM')
model = Sequential()
model.add(
LSTM(input_shape=(None, feats.input_shape[1]),
units=units[0],
return_sequences=True))
if batch_norm:
model.add(BatchNormalization())
model.add(Activation('relu'))
if dropout:
model.add(Dropout(dropout))
if len(units) > 2:
for unit in units[1:-1]:
model.add(LSTM(units=unit, return_sequences=True))
if batch_norm:
model.add(BatchNormalization())
model.add(Activation('relu'))
if dropout:
model.add(Dropout(dropout))
model.add(LSTM(units=units[-1], return_sequences=False))
if batch_norm:
model.add(BatchNormalization())
model.add(Activation('relu'))
if dropout:
model.add(Dropout(dropout))
model.add(Dense(units=feats.num_classes))
model.add(Activation("softmax"))
##---DenseFeedforward Network
#Makes a hidden layer for each item in units
if feats.model_type == 'NN':
model = Sequential()
model.add(Flatten(input_shape=feats.input_shape))
for unit in units:
model.add(Dense(unit))
if batch_norm:
model.add(BatchNormalization())
model.add(Activation('relu'))
if dropout:
model.add(Dropout(dropout))
model.add(Dense(feats.num_classes, activation='softmax'))
##----Convolutional Network
if feats.model_type == 'CNN':
if nunits < 2:
print('Warning: Need at least two layers for CNN')
model = Sequential()
model.add(
Conv2D(units[0],
filt_size,
input_shape=feats.input_shape,
padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=pool_size, padding='same'))
if nunits > 2:
for unit in units[1:-1]:
model.add(Conv2D(unit, filt_size, padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=pool_size, padding='same'))
model.add(Flatten())
model.add(Dense(units[-1]))
model.add(Activation('relu'))
model.add(Dense(feats.num_classes))
model.add(Activation('softmax'))
##----Convolutional Network
if feats.model_type == 'CNN3D':
if nunits < 2:
print('Warning: Need at least two layers for CNN')
model = Sequential()
model.add(
Conv3D(units[0],
filt_size,
input_shape=feats.input_shape,
padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling3D(pool_size=pool_size, padding='same'))
if nunits > 2:
for unit in units[1:-1]:
model.add(Conv3D(unit, filt_size, padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling3D(pool_size=pool_size, padding='same'))
model.add(Flatten())
model.add(Dense(units[-1]))
model.add(Activation('relu'))
model.add(Dense(feats.num_classes))
model.add(Activation('softmax'))
## Autoencoder
#takes the first item in units for hidden layer size
if feats.model_type == 'AUTO':
encoding_dim = units[0]
input_data = Input(shape=(feats.input_shape[0], ))
#,activity_regularizer=regularizers.l1(10e-5)
encoded = Dense(encoding_dim, activation='relu')(input_data)
decoded = Dense(feats.input_shape[0], activation='sigmoid')(encoded)
model = Model(input_data, decoded)
encoder = Model(input_data, encoded)
encoded_input = Input(shape=(encoding_dim, ))
decoder_layer = model.layers[-1]
decoder = Model(encoded_input, decoder_layer(encoded_input))
#takes an odd number of layers > 1
#e.g. units = [64,32,16,32,64]
if feats.model_type == 'AUTODeep':
if nunits % 2 == 0:
print('Warning: Please enter odd number of layers into units')
half = nunits / 2
midi = int(np.floor(half))
input_data = Input(shape=(feats.input_shape[0], ))
encoded = Dense(units[0], activation='relu')(input_data)
#encoder decreases
if nunits >= 3:
for unit in units[1:midi]:
encoded = Dense(unit, activation='relu')(encoded)
#latent space
decoded = Dense(units[midi], activation='relu')(encoded)
#decoder increses
if nunits >= 3:
for unit in units[midi + 1:-1]:
decoded = Dense(unit, activation='relu')(decoded)
decoded = Dense(units[-1], activation='relu')(decoded)
decoded = Dense(feats.input_shape[0], activation='sigmoid')(decoded)
model = Model(input_data, decoded)
encoder = Model(input_data, encoded)
encoded_input = Input(shape=(units[midi], ))
if feats.model_type == 'AUTO' or feats.model_type == 'AUTODeep':
opt = tf.keras.optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
model.compile(optimizer=opt, loss='mean_squared_error')
if ((feats.model_type == 'CNN') or (feats.model_type == 'CNN3D')
or (feats.model_type == 'LSTM') or (feats.model_type == 'NN')):
# initiate adam optimizer
opt = tf.keras.optimizers.Adam(lr=0.01,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
# Let's train the model using RMSprop
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['acc'])
encoder = []
model.summary()
return model, encoder
(n_samples, 3, 16, 16, 16)).astype(np.float32)
# (B, C, D, H, W) -> (B, D, H, W, C)
x_train = np.transpose(x_train, (0, 2, 3, 4, 1)) # Set channel last
# Shape: (n_samples, dim=2)
y_train = np.linspace(0, 1, n_samples * num_classes).reshape(
(n_samples, num_classes)).astype(np.float32)
x_test, y_test = x_train, y_train
print("Train data shape:", x_train.shape)
print("Train labels shape:", y_train.shape)
print("Test data shape:", x_test.shape)
print("Test labels shape:", y_test.shape)
model = Sequential()
model.add(Input(shape=(16, 16, 16, 3), name="linput"))
model.add(Conv3D(5, 3, padding="same"))
model.add(MaxPooling3D())
model.add(Conv3D(10, 3, padding="same"))
model.add(MaxPooling3D())
model.add(Flatten())
model.add(Dense(100))
model.add(Dense(num_classes))
model.build(input_shape=(16, 16, 16, 3)) # For keras2onnx
model.compile(loss='mse', optimizer="adam")
model.summary()
# Training
model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs)
def _build_model(self):
# ------------------ build evaluate_net ------------------
# shared part
shared_model = models.Sequential()
# pre-process block
shared_model.add(
Conv3D(32, (3, 3, 3),
strides=(1, 1, 1),
input_shape=self.input_shape,
name='conv1'))
shared_model.add(BatchNormalization(name='b1'))
shared_model.add(Activation('relu'))
shared_model.add(
MaxPooling3D(pool_size=(2, 2, 1),
strides=1,
padding="VALID",
name='p1'))
# resnet blocks
shared_model.add(self.build_resblock(32, 2, name='Resnet_1'))
shared_model.add(self.build_resblock(64, 2, name='Resnet_2', stride=2))
shared_model.add(self.build_resblock(64, 2, name='Resnet_3', stride=2))
shared_model.add(self.build_resblock(128, 2, name='Resnet_4',
stride=2))
# shared_model.summary()
self.shared_model = shared_model
# action model
act_model = models.Sequential()
# add shared block
act_model.add(shared_model)
# fully connected block
act_model.add(GlobalAveragePooling3D())
act_model.add(
Dense(self.act_dim,
name="d1",
kernel_regularizer=regularizers.L2(0.001)))
# act_model.summary()
self.act_model = act_model
# move model
move_model = models.Sequential()
# add shared block
move_model.add(shared_model)
# fully connected block
move_model.add(GlobalAveragePooling3D())
move_model.add(
Dense(4, name="d1", kernel_regularizer=regularizers.L2(0.001)))
# move_model.summary()
self.move_model = move_model
# ------------------ build target_model ------------------
shared_target_model = models.Sequential()
# pre-process block
shared_target_model.add(
Conv3D(32, (3, 3, 3),
strides=(1, 1, 1),
input_shape=self.input_shape,
name='conv1'))
shared_target_model.add(BatchNormalization(name='b1'))
shared_target_model.add(Activation('relu'))
shared_target_model.add(
MaxPooling3D(pool_size=(2, 2, 1),
strides=1,
padding="VALID",
name='p1'))
# resnet blocks
shared_target_model.add(self.build_resblock(32, 2, name='Resnet_1'))
shared_target_model.add(
self.build_resblock(64, 2, name='Resnet_2', stride=2))
shared_target_model.add(
self.build_resblock(64, 2, name='Resnet_3', stride=2))
shared_target_model.add(
self.build_resblock(128, 2, name='Resnet_4', stride=2))
self.shared_target_model = shared_target_model
# action model
act_target_model = models.Sequential()
# add shared block
act_target_model.add(shared_target_model)
# fully connected block
act_target_model.add(GlobalAveragePooling3D())
act_target_model.add(
Dense(self.act_dim,
name="d1",
kernel_regularizer=regularizers.L2(0.001)))
# act_target_model.summary()
self.act_target_model = act_target_model
# move model
move_target_model = models.Sequential()
# add shared block
move_target_model.add(shared_target_model)
# fully connected block
move_target_model.add(GlobalAveragePooling3D())
move_target_model.add(
Dense(4, name="d1", kernel_regularizer=regularizers.L2(0.001)))
# move_target_model.summary()
self.move_target_model = move_target_model
def getCoarse2FineModel(summary=True):
tf.debugging.set_log_device_placement(True)
# defined input
videoclip_cropped = Input((c, t, h, w), name='input1')
videoclip_original = Input((c, t, h, w), name='input2')
last_frame_bigger = Input((c, h * upsample, w * upsample), name='input3')
# coarse saliency model
coarse_saliency_model = Sequential()
coarse_saliency_model.add(
Conv3D(64, [3, 3, 3],
activation='relu',
padding='same',
name='conv1',
strides=(1, 1, 1),
data_format='channels_first'))
coarse_saliency_model.add(
MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
padding='valid',
name='pool1',
data_format='channels_first'))
coarse_saliency_model.add(
Conv3D(128, [3, 3, 3],
activation='relu',
padding='same',
name='conv2',
strides=(1, 1, 1),
data_format='channels_first'))
coarse_saliency_model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
padding='valid',
name='pool2',
data_format='channels_first'))
coarse_saliency_model.add(
Conv3D(256, [3, 3, 3],
activation='relu',
padding='same',
name='conv3a',
strides=(1, 1, 1),
data_format='channels_first'))
coarse_saliency_model.add(
Conv3D(256, [3, 3, 3],
activation='relu',
padding='same',
name='conv3b',
strides=(1, 1, 1),
data_format='channels_first'))
coarse_saliency_model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
padding='valid',
name='pool3',
data_format='channels_first'))
coarse_saliency_model.add(
Conv3D(512, [3, 3, 3],
activation='relu',
padding='same',
name='conv4a',
strides=(1, 1, 1),
data_format='channels_first'))
coarse_saliency_model.add(
Conv3D(512, [3, 3, 3],
activation='relu',
padding='same',
name='conv4b',
strides=(1, 1, 1),
data_format='channels_first'))
coarse_saliency_model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(4, 2, 2),
padding='valid',
name='pool4',
data_format='channels_first'))
coarse_saliency_model.add(Reshape((512, 7, 7)))
coarse_saliency_model.add(BatchNormalization())
coarse_saliency_model.add(
Conv2D(256, [3, 3],
kernel_initializer='glorot_uniform',
padding='same',
data_format='channels_first'))
coarse_saliency_model.add(LeakyReLU(alpha=.001))
coarse_saliency_model.add(
UpSampling2D(size=(2, 2), data_format='channels_first'))
coarse_saliency_model.add(BatchNormalization())
coarse_saliency_model.add(
Conv2D(128, [3, 3],
kernel_initializer='glorot_uniform',
padding='same',
data_format='channels_first'))
coarse_saliency_model.add(LeakyReLU(alpha=.001))
coarse_saliency_model.add(
UpSampling2D(size=(2, 2), data_format='channels_first'))
coarse_saliency_model.add(BatchNormalization())
coarse_saliency_model.add(
Conv2D(64, [3, 3],
kernel_initializer='glorot_uniform',
padding='same',
data_format='channels_first'))
coarse_saliency_model.add(LeakyReLU(alpha=.001))
coarse_saliency_model.add(
UpSampling2D(size=(2, 2), data_format='channels_first'))
coarse_saliency_model.add(BatchNormalization())
coarse_saliency_model.add(
Conv2D(32, [3, 3],
kernel_initializer='glorot_uniform',
padding='same',
data_format='channels_first'))
coarse_saliency_model.add(LeakyReLU(alpha=.001))
coarse_saliency_model.add(
UpSampling2D(size=(2, 2), data_format='channels_first'))
coarse_saliency_model.add(BatchNormalization())
coarse_saliency_model.add(
Conv2D(16, [3, 3],
kernel_initializer='glorot_uniform',
padding='same',
data_format='channels_first'))
coarse_saliency_model.add(LeakyReLU(alpha=.001))
coarse_saliency_model.add(BatchNormalization())
coarse_saliency_model.add(
Conv2D(1, [3, 3],
kernel_initializer='glorot_uniform',
padding='same',
data_format='channels_first'))
coarse_saliency_model.add(LeakyReLU(alpha=.001))
# loss on cropped image
coarse_saliency_cropped = coarse_saliency_model(videoclip_cropped)
cropped_output = Flatten(name='cropped_output')(coarse_saliency_cropped)
# coarse-to-fine saliency model and loss
coarse_saliency_original = coarse_saliency_model(videoclip_original)
x = UpSampling2D((upsample, upsample),
name='coarse_saliency_upsampled',
data_format='channels_first')(
coarse_saliency_original) # 112 x 4 = 448
x = concatenate([x, last_frame_bigger], axis=1) # merge the last RGB frame
x = Conv2D(32, [3, 3],
padding='same',
kernel_initializer='he_normal',
data_format='channels_first')(x)
x = Conv2D(64, [3, 3],
padding='same',
kernel_initializer='he_normal',
data_format='channels_first')(x)
x = LeakyReLU(alpha=.001)(x)
x = Conv2D(32, [3, 3],
padding='same',
kernel_initializer='he_normal',
data_format='channels_first')(x)
x = LeakyReLU(alpha=.001)(x)
x = Conv2D(32, [3, 3],
padding='same',
kernel_initializer='he_normal',
data_format='channels_first')(x)
x = LeakyReLU(alpha=.001)(x)
x = Conv2D(16, [3, 3],
padding='same',
kernel_initializer='he_normal',
data_format='channels_first')(x)
x = LeakyReLU(alpha=.001)(x)
x = Conv2D(4, [3, 3],
padding='same',
kernel_initializer='he_normal',
data_format='channels_first')(x)
x = LeakyReLU(alpha=.001)(x)
fine_saliency_model = Conv2D(1, [3, 3],
padding='same',
activation='relu',
data_format='channels_first')(x)
# loss on full image
full_fine_output = Flatten(name='full_fine_output')(fine_saliency_model)
final_model = Model(
inputs=[videoclip_cropped, videoclip_original, last_frame_bigger],
outputs=[cropped_output, full_fine_output])
if summary:
print(final_model.summary())
return final_model
def residual_block(x, f_maps, filter_size, activation='elu', k_init='he_normal',
drop_value=0.0, bn=False, first_block=False):
"""Residual block.
Parameters
----------
x : Keras layer
iInput layer of the block.
f_maps : array of ints
Feature maps to use.
filter_size : 3 int tuple
Height, width and depth of the convolution window.
activation : str, optional
Keras available activation type.
k_init : str, optional
Keras available kernel initializer type.
drop_value : float, optional
Dropout value to be fixed.
bn : bool, optional
Use batch normalization.
first_block : float, optional
To advice the function that it is the first residual block of the
network, which avoids Full Pre-Activation layers.
Returns
-------
x : Keras layer
Last layer of the block.
"""
# Create shorcut
shortcut = Conv3D(f_maps, activation=None, kernel_size=(1, 1, 1),
kernel_initializer=k_init)(x)
# Main path
if not first_block:
x = BatchNormalization()(x) if bn else x
if activation == "elu":
x = ELU(alpha=1.0) (x)
x = Conv3D(f_maps, filter_size, activation=None,
kernel_initializer=k_init, padding='same') (x)
x = Dropout(drop_value) (x) if drop_value > 0 else x
x = BatchNormalization()(x) if bn else x
if activation == "elu":
x = ELU(alpha=1.0) (x)
x = Conv3D(f_maps, filter_size, activation=None,
kernel_initializer=k_init, padding='same') (x)
x = BatchNormalization()(x) if bn else x
# Add shortcut value to main path
x = Add()([shortcut, x])
return x
def new_model(X_train,
y_train,
X_valid=None,
y_valid=None,
final=False,
out=2,
dr=0.02,
lr=0.00001,
breg=l2(0.0001),
areg=None,
n_epochs=30,
batch_size=15):
dim = (64, 64, 64, 1)
model = Sequential()
model.add(
Conv3D(32,
kernel_size=(5, 5, 5),
kernel_initializer='he_uniform',
bias_regularizer=breg,
input_shape=dim))
model.add(Activation('relu'))
model.add(MaxPooling3D(pool_size=(2, 2, 2)))
model.add(
Conv3D(64,
kernel_size=(5, 5, 5),
bias_regularizer=breg,
kernel_initializer='he_uniform'))
model.add(Activation('relu'))
model.add(MaxPooling3D(pool_size=(2, 2, 2)))
model.add(
Conv3D(128,
kernel_size=(5, 5, 5),
bias_regularizer=breg,
kernel_initializer='he_uniform'))
model.add(Activation('relu'))
model.add(MaxPooling3D(pool_size=(2, 2, 2)))
model.add(Dropout(dr))
model.add(Flatten())
model.add(
Dense(512, bias_regularizer=breg, kernel_initializer='he_uniform'))
model.add(Activation('relu'))
model.add(Dropout(dr))
model.add(
Dense(256, bias_regularizer=breg, kernel_initializer='he_uniform'))
model.add(Activation('relu'))
model.add(Dense(out, activation='softmax', activity_regularizer=areg))
opt = Adam(learning_rate=lr)
model.compile(optimizer=opt,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
print("model")
model.summary()
cb = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=5,
verbose=1,
epsilon=1e-4,
mode='min')
if not final:
hist = model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=n_epochs,
callbacks=[cb],
validation_data=(X_valid, y_valid),
shuffle=True)
# model final training for testing (train + valid combined)
else:
hist = model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=n_epochs,
callbacks=[cb],
shuffle=True)
return model, hist
def load_model(input_shape,
num_labels,
axis=-1,
base_filter=32,
depth_size=4,
se_res_block=True,
se_ratio=16,
noise=0.1,
last_relu=False,
atten_gate=False):
def conv3d(layer_input,
filters,
axis=-1,
se_res_block=True,
se_ratio=16,
down_sizing=True):
if down_sizing == True:
layer_input = MaxPooling3D(pool_size=(2, 2, 2))(layer_input)
d = Conv3D(filters, (3, 3, 3), use_bias=False,
padding='same')(layer_input)
d = InstanceNormalization(axis=axis)(d)
d = LeakyReLU(alpha=0.3)(d)
d = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(d)
d = InstanceNormalization(axis=axis)(d)
if se_res_block == True:
se = GlobalAveragePooling3D()(d)
se = Dense(filters // se_ratio, activation='relu')(se)
se = Dense(filters, activation='sigmoid')(se)
se = Reshape([1, 1, 1, filters])(se)
d = Multiply()([d, se])
shortcut = Conv3D(filters, (3, 3, 3),
use_bias=False,
padding='same')(layer_input)
shortcut = InstanceNormalization(axis=axis)(shortcut)
d = add([d, shortcut])
d = LeakyReLU(alpha=0.3)(d)
return d
def deconv3d(layer_input,
skip_input,
filters,
axis=-1,
se_res_block=True,
se_ratio=16,
atten_gate=False):
if atten_gate == True:
gating = Conv3D(filters, (1, 1, 1), use_bias=False,
padding='same')(layer_input)
gating = InstanceNormalization(axis=axis)(gating)
attention = Conv3D(filters, (2, 2, 2),
strides=(2, 2, 2),
use_bias=False,
padding='valid')(skip_input)
attention = InstanceNormalization(axis=axis)(attention)
attention = add([gating, attention])
attention = Conv3D(1, (1, 1, 1),
use_bias=False,
padding='same',
activation='sigmoid')(attention)
# attention = Lambda(resize_by_axis, arguments={'dim_1':skip_input.get_shape().as_list()[1],'dim_2':skip_input.get_shape().as_list()[2],'ax':3})(attention) # error when None dimension is feeded.
# attention = Lambda(resize_by_axis, arguments={'dim_1':skip_input.get_shape().as_list()[1],'dim_2':skip_input.get_shape().as_list()[3],'ax':2})(attention)
attention = ZeroPadding3D(((0, 1), (0, 1), (0, 1)))(attention)
attention = UpSampling3D((2, 2, 2))(attention)
attention = CropToConcat3D(mode='crop')([attention, skip_input])
attention = Lambda(lambda x: K.tile(x, [1, 1, 1, 1, filters]))(
attention)
skip_input = multiply([skip_input, attention])
u1 = ZeroPadding3D(((0, 1), (0, 1), (0, 1)))(layer_input)
u1 = Conv3DTranspose(filters, (2, 2, 2),
strides=(2, 2, 2),
use_bias=False,
padding='same')(u1)
u1 = InstanceNormalization(axis=axis)(u1)
u1 = LeakyReLU(alpha=0.3)(u1)
u1 = CropToConcat3D()([u1, skip_input])
u2 = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(u1)
u2 = InstanceNormalization(axis=axis)(u2)
u2 = LeakyReLU(alpha=0.3)(u2)
u2 = Conv3D(filters, (3, 3, 3), use_bias=False, padding='same')(u2)
u2 = InstanceNormalization(axis=axis)(u2)
if se_res_block == True:
se = GlobalAveragePooling3D()(u2)
se = Dense(filters // se_ratio, activation='relu')(se)
se = Dense(filters, activation='sigmoid')(se)
se = Reshape([1, 1, 1, filters])(se)
u2 = Multiply()([u2, se])
shortcut = Conv3D(filters, (3, 3, 3),
use_bias=False,
padding='same')(u1)
shortcut = InstanceNormalization(axis=axis)(shortcut)
u2 = add([u2, shortcut])
u2 = LeakyReLU(alpha=0.3)(u2)
return u2
def CropToConcat3D(mode='concat'):
def crop_to_concat_3D(concat_layers, axis=-1):
bigger_input, smaller_input = concat_layers
bigger_shape, smaller_shape = tf.shape(bigger_input), \
tf.shape(smaller_input)
sh, sw, sd = smaller_shape[1], smaller_shape[2], smaller_shape[3]
bh, bw, bd = bigger_shape[1], bigger_shape[2], bigger_shape[3]
dh, dw, dd = bh - sh, bw - sw, bd - sd
cropped_to_smaller_input = bigger_input[:, :-dh, :-dw, :-dd, :]
if mode == 'concat':
return K.concatenate([smaller_input, cropped_to_smaller_input],
axis=axis)
elif mode == 'add':
return smaller_input + cropped_to_smaller_input
elif mode == 'crop':
return cropped_to_smaller_input
return Lambda(crop_to_concat_3D)
def resize_by_axis(image, dim_1, dim_2,
ax): # it is available only for 1 channel 3D
resized_list = []
unstack_img_depth_list = tf.unstack(image, axis=ax)
for i in unstack_img_depth_list:
resized_list.append(tf.image.resize_images(
i, [dim_1, dim_2])) # defaults to ResizeMethod.BILINEAR
stack_img = tf.stack(resized_list, axis=ax + 1)
return stack_img
input_img = Input(shape=input_shape, name='Input')
d0 = GaussianNoise(noise)(input_img)
d1 = Conv3D(base_filter, (3, 3, 3), use_bias=False, padding='same')(d0)
d1 = InstanceNormalization(axis=axis)(d1)
d1 = LeakyReLU(alpha=0.3)(d1)
d2 = conv3d(d1, base_filter * 2, se_res_block=se_res_block)
d3 = conv3d(d2, base_filter * 4, se_res_block=se_res_block)
d4 = conv3d(d3, base_filter * 8, se_res_block=se_res_block)
if depth_size == 4:
d5 = conv3d(d4, base_filter * 16, se_res_block=se_res_block)
u4 = deconv3d(d5,
d4,
base_filter * 8,
se_res_block=se_res_block,
atten_gate=atten_gate)
u3 = deconv3d(u4,
d3,
base_filter * 4,
se_res_block=se_res_block,
atten_gate=atten_gate)
elif depth_size == 3:
u3 = deconv3d(d4,
d3,
base_filter * 4,
se_res_block=se_res_block,
atten_gate=atten_gate)
else:
raise Exception('depth size must be 3 or 4. you put ', depth_size)
u2 = deconv3d(u3,
d2,
base_filter * 2,
se_res_block=se_res_block,
atten_gate=atten_gate)
u1 = ZeroPadding3D(((0, 1), (0, 1), (0, 1)))(u2)
u1 = Conv3DTranspose(base_filter, (2, 2, 2),
strides=(2, 2, 2),
use_bias=False,
padding='same')(u1)
u1 = InstanceNormalization(axis=axis)(u1)
u1 = LeakyReLU(alpha=0.3)(u1)
u1 = CropToConcat3D()([u1, d1])
u1 = Conv3D(base_filter, (3, 3, 3), use_bias=False, padding='same')(u1)
u1 = InstanceNormalization(axis=axis)(u1)
u1 = LeakyReLU(alpha=0.3)(u1)
output_img = Conv3D(num_labels,
kernel_size=1,
strides=1,
padding='same',
activation='sigmoid')(u1)
if last_relu == True:
output_img = ThresholdedReLU(theta=0.5)(output_img)
model = Model(inputs=input_img, outputs=output_img)
return model
def U_Net_3D(image_shape,
activation='elu',
feature_maps=[32, 64, 128, 256],
depth=3,
drop_values=[0.1, 0.1, 0.1, 0.1],
spatial_dropout=False,
batch_norm=False,
k_init='he_normal',
z_down=2,
loss_type="bce",
optimizer="sgd",
lr=0.001,
n_classes=1):
"""Create 3D U-Net.
Parameters
----------
image_shape : 3D tuple
Dimensions of the input image.
activation : str, optional
Keras available activation type.
feature_maps : array of ints, optional
Feature maps to use on each level. Must have the same length as the
``depth+1``.
depth : int, optional
Depth of the network.
drop_values : float, optional
Dropout value to be fixed.
spatial_dropout : bool, optional
Use spatial dropout instead of the `normal` dropout.
batch_norm : bool, optional
Make batch normalization.
k_init : string, optional
Kernel initialization for convolutional layers.
z_down : int, optional
Downsampling used in z dimension. Set it to 1 if the dataset is not
isotropic.
loss_type : str, optional
Loss type to use, three type available: ``bce`` (Binary Cross Entropy)
, ``w_bce`` (Weighted BCE, based on weigth maps) and ``w_bce_dice``
(Weighted loss: ``weight1*BCE + weight2*Dice``).
optimizer : str, optional
Optimizer used to minimize the loss function. Posible options: ``sgd``
or ``adam``.
lr : float, optional
Learning rate value.
n_classes: int, optional
Number of classes.
Returns
-------
model : Keras model
Model containing the U-Net.
Calling this function with its default parameters returns the following
network:
.. image:: img/unet_3d.png
:width: 100%
:align: center
Image created with `PlotNeuralNet <https://github.com/HarisIqbal88/PlotNeuralNet>`_.
"""
if len(feature_maps) != depth + 1:
raise ValueError("feature_maps dimension must be equal depth+1")
if len(drop_values) != depth + 1:
raise ValueError("'drop_values' dimension must be equal depth+1")
dinamic_dim = (None, ) * (len(image_shape) - 1) + (image_shape[-1], )
inputs = Input(dinamic_dim)
x = inputs
#x = Input(image_shape)
#inputs = x
if loss_type == "w_bce":
weights = Input(image_shape)
# List used to access layers easily to make the skip connections of the U-Net
l = []
# ENCODER
for i in range(depth):
x = Conv3D(feature_maps[i], (3, 3, 3),
activation=None,
kernel_initializer=k_init,
padding='same')(x)
x = BatchNormalization()(x) if batch_norm else x
x = Activation(activation)(x)
if spatial_dropout and drop_values[i] > 0:
x = SpatialDropout3D(drop_values[i])(x)
elif drop_values[i] > 0 and not spatial_dropout:
x = Dropout(drop_values[i])(x)
x = Conv3D(feature_maps[i], (3, 3, 3),
activation=None,
kernel_initializer=k_init,
padding='same')(x)
x = BatchNormalization()(x) if batch_norm else x
x = Activation(activation)(x)
l.append(x)
x = MaxPooling3D((2, 2, z_down))(x)
# BOTTLENECK
x = Conv3D(feature_maps[depth], (3, 3, 3),
activation=None,
kernel_initializer=k_init,
padding='same')(x)
x = BatchNormalization()(x) if batch_norm else x
x = Activation(activation)(x)
if spatial_dropout and drop_values[depth] > 0:
x = SpatialDropout3D(drop_values[depth])(x)
elif drop_values[depth] > 0 and not spatial_dropout:
x = Dropout(drop_values[depth])(x)
x = Conv3D(feature_maps[depth], (3, 3, 3),
activation=None,
kernel_initializer=k_init,
padding='same')(x)
x = BatchNormalization()(x) if batch_norm else x
x = Activation(activation)(x)
# DECODER
for i in range(depth - 1, -1, -1):
x = Conv3DTranspose(feature_maps[i], (2, 2, 2),
strides=(2, 2, z_down),
padding='same')(x)
x = concatenate([x, l[i]])
x = Conv3D(feature_maps[i], (3, 3, 3),
activation=None,
kernel_initializer=k_init,
padding='same')(x)
x = BatchNormalization()(x) if batch_norm else x
x = Activation(activation)(x)
if spatial_dropout and drop_values[i] > 0:
x = SpatialDropout3D(drop_values[i])(x)
elif drop_values[i] > 0 and not spatial_dropout:
x = Dropout(drop_values[i])(x)
x = Conv3D(feature_maps[i], (3, 3, 3),
activation=None,
kernel_initializer=k_init,
padding='same')(x)
x = BatchNormalization()(x) if batch_norm else x
x = Activation(activation)(x)
outputs = Conv3D(n_classes, (1, 1, 1), activation='sigmoid')(x)
# Loss type
if loss_type == "w_bce":
model = Model(inputs=[inputs, weights], outputs=[outputs])
else:
model = Model(inputs=[inputs], outputs=[outputs])
# Select the optimizer
if optimizer == "sgd":
opt = tf.keras.optimizers.SGD(lr=lr,
momentum=0.99,
decay=0.0,
nesterov=False)
elif optimizer == "adam":
opt = tf.keras.optimizers.Adam(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
else:
raise ValueError("Error: optimizer value must be 'sgd' or 'adam'")
# Compile the model
if loss_type == "bce":
if n_classes > 1:
model.compile(optimizer=opt,
loss='categorical_crossentropy',
metrics=[jaccard_index_softmax])
else:
model.compile(optimizer=opt,
loss='binary_crossentropy',
metrics=[jaccard_index])
elif loss_type == "w_bce":
model.compile(optimizer=opt,
loss=binary_crossentropy_weighted(weights),
metrics=[jaccard_index])
elif loss_type == "w_bce_dice":
model.compile(optimizer=opt,
loss=weighted_bce_dice_loss(w_dice=0.66, w_bce=0.33),
metrics=[jaccard_index])
else:
raise ValueError("'loss_type' must be 'bce', 'w_bce' or 'w_bce_dice'")
return model
def U_Net_3D_Xiao(image_shape, lr=0.0001, n_classes=2):
"""Create 3D U-Net.
Parameters
----------
image_shape : array of 3 int
Dimensions of the input image.
activation : str, optional
Keras available activation type.
lr : float, optional
Learning rate value.
n_classes : int, optional
Number of classes.
Returns
-------
model : Keras model
Xiao 3D U-Net type proposed network model.
Here is a picture of the network extracted from the original paper:
.. image:: img/xiao_network.jpg
:width: 100%
:align: center
"""
dinamic_dim = (None, ) * (len(image_shape) - 1) + (1, )
inputs = Input(dinamic_dim)
x = Conv3D(3, (3, 3, 3),
activation=None,
kernel_initializer='he_normal',
padding='same')(inputs)
x = BatchNormalization()(x)
x = ELU()(x)
# Encoder
s1 = residual_block(x, 32)
x = MaxPooling3D((2, 2, 1), padding='same')(s1)
s2 = residual_block(x, 48)
x = MaxPooling3D((2, 2, 1), padding='same')(s2)
s3 = residual_block(x, 64)
x = MaxPooling3D((2, 2, 1), padding='same')(s3)
x = residual_block(x, 80)
# Decoder
x = UpSampling3D((2, 2, 1))(x)
x = Conv3D(64, (1, 1, 1),
activation=None,
kernel_initializer='he_normal',
padding='same')(x)
x = BatchNormalization()(x)
x = ELU()(x)
x = Add()([s3, x])
x = residual_block(x, 64)
x = UpSampling3D((2, 2, 1))(x)
# Auxiliary ouput 1
a1 = UpSampling3D((2, 2, 1))(x)
a1 = Conv3D(n_classes, (1, 1, 1),
activation=None,
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01),
name='aux1')(a1)
a1 = Activation('softmax')(a1)
x = Conv3D(48, (1, 1, 1),
activation=None,
kernel_initializer='he_normal',
padding='same')(x)
x = BatchNormalization()(x)
x = ELU()(x)
x = Add()([s2, x])
x = residual_block(x, 48)
x = UpSampling3D((2, 2, 1))(x)
x = Conv3D(32, (1, 1, 1),
activation=None,
kernel_initializer='he_normal',
padding='same')(x)
x = BatchNormalization()(x)
x = ELU()(x)
# Auxiliary ouput 2
a2 = Conv3D(n_classes, (1, 1, 1),
activation=None,
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01),
name='aux2')(x)
a2 = Activation('softmax')(a2)
x = Add()([s1, x])
x = residual_block(x, 32)
x = Conv3D(3, (3, 3, 3),
activation=None,
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01))(x)
x = BatchNormalization()(x)
x = ELU()(x)
# Adapt the output to use softmax pixel-wise
x = Conv3D(n_classes, (1, 1, 1),
activation=None,
kernel_initializer='he_normal',
padding='same')(x)
outputs = Activation('softmax', name='main_classifier')(x)
model = Model(inputs=[inputs], outputs=[a1, a2, outputs])
opt = tf.keras.optimizers.Adam(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-8,
decay=0.0,
amsgrad=False)
model.compile(optimizer=opt,
loss="categorical_crossentropy",
loss_weights=[0.15, 0.3, 1],
metrics=[jaccard_index_softmax])
return model
def build_generator(self, number_of_filters=64):
def build_residual_block(input, number_of_filters, kernel_size=3):
shortcut = input
if self.dimensionality == 2:
input = Conv2D(filters=number_of_filters,
kernel_size=kernel_size,
strides=1,
padding='same')(input)
else:
input = Conv3D(filters=number_of_filters,
kernel_size=kernel_size,
strides=1,
padding='same')(input)
input = ReLU()(input)
input = BatchNormalization(momentum=0.8)(input)
if self.dimensionality == 2:
input = Conv2D(filters=number_of_filters,
kernel_size=kernel_size,
strides=1,
padding='same')(input)
else:
input = Conv3D(filters=number_of_filters,
kernel_size=kernel_size,
strides=1,
padding='same')(input)
input = BatchNormalization(momentum=0.8)(input)
input = Add()([input, shortcut])
return (input)
def build_deconvolution_layer(input,
number_of_filters=256,
kernel_size=3):
model = input
if self.dimensionality == 2:
model = UpSampling2D(size=2)(model)
input = Conv2D(filters=number_of_filters,
kernel_size=kernel_size,
strides=1,
padding='same')(model)
else:
model = UpSampling3D(size=2)(model)
input = Conv3D(filters=number_of_filters,
kernel_size=kernel_size,
strides=1,
padding='same')(model)
model = ReLU()(model)
return (model)
image = Input(shape=self.low_resolution_image_size)
pre_residual = image
if self.dimensionality == 2:
pre_residual = Conv2D(filters=number_of_filters,
kernel_size=9,
strides=1,
padding='same')(pre_residual)
else:
pre_residual = Conv3D(filters=number_of_filters,
kernel_size=9,
strides=1,
padding='same')(pre_residual)
residuals = build_residual_block(
pre_residual,
number_of_filters=self.number_of_filters_at_base_layer[0])
for i in range(self.number_of_residual_blocks - 1):
residuals = build_residual_block(
residuals,
number_of_filters=self.number_of_filters_at_base_layer[0])
post_residual = residuals
if self.dimensionality == 2:
post_residual = Conv2D(filters=number_of_filters,
kernel_size=3,
strides=1,
padding='same')(post_residual)
else:
post_residual = Conv3D(filters=number_of_filters,
kernel_size=3,
strides=1,
padding='same')(post_residual)
post_residual = BatchNormalization(momentum=0.8)(post_residual)
model = Add()([post_residual, pre_residual])
# upsampling
if self.scale_factor >= 2:
model = build_deconvolution_layer(model)
if self.scale_factor >= 4:
model = build_deconvolution_layer(model)
if self.scale_factor == 8:
model = build_deconvolution_layer(model)
if self.dimensionality == 2:
model = Conv2D(filters=self.number_of_channels,
kernel_size=9,
strides=1,
padding='same',
activation='tanh')(model)
else:
model = Conv3D(filters=self.number_of_channels,
kernel_size=9,
strides=1,
padding='same',
activation='tanh')(model)
generator = Model(inputs=image, outputs=model)
return (generator)
def create(self):
input_layer = Input(shape=eval(self.config.input.shape),
name='input_layer')
# First conv layer
conv_1 = Conv3D(filters=8,
kernel_size=3,
dilation_rate=1,
strides=2,
activation=LeakyReLU(0.2),
kernel_initializer=self._he_normal_initializer,
kernel_regularizer=self._kernel_regularizer,
name='conv_1')(input_layer)
batch_norm_1 = batch_norm_1 = BatchNormalization(
name='batch_norm_1')(conv_1)
# Second conv layer
conv_2 = Conv3D(filters=8,
kernel_size=3,
dilation_rate=1,
strides=2,
activation=LeakyReLU(0.2),
kernel_initializer=self._he_normal_initializer,
kernel_regularizer=self._kernel_regularizer,
name='conv_2')(batch_norm_1)
batch_norm_2 = BatchNormalization(name='batch_norm_2')(conv_2)
# Third conv layer
conv_3 = Conv3D(filters=8,
kernel_size=3,
dilation_rate=1,
strides=2,
activation=LeakyReLU(0.2),
kernel_initializer=self._he_normal_initializer,
kernel_regularizer=self._kernel_regularizer,
name='conv_3')(batch_norm_2)
batch_norm_3 = BatchNormalization(name='batch_norm_3')(conv_3)
# Flatten
flatten = Flatten(name='flatten')(batch_norm_3)
# Dense layer 1
dense_1 = Dense(1024,
activation=LeakyReLU(),
kernel_initializer=self._he_normal_initializer,
kernel_regularizer=self._kernel_regularizer,
name='dense_1')(flatten)
drop_1 = Dropout(0.5, name='dropout_1')(dense_1)
# Dense layer 2
dense_2 = Dense(512,
activation=LeakyReLU(),
kernel_initializer=self._he_normal_initializer,
kernel_regularizer=self._kernel_regularizer,
name='dense_2')(drop_1)
drop_2 = Dropout(0.5, name='dropout_2')(dense_2)
# Output layer
output_layer = Dense(3,
activation='softmax',
kernel_initializer=self._lecun_normal_initializer,
name='output')(drop_2)
model = keras_model(input_layer, output_layer)
model.compile(loss='categorical_crossentropy',
optimizer=self._optimizer,
metrics=[tf.keras.metrics.CategoricalAccuracy()])
return model
def ResUNet_3D(image_shape, activation='elu', k_init='he_normal',
drop_values=[0.1,0.1,0.1,0.1,0.1], batch_norm=False,
feature_maps=[16,32,64,128,256], depth=4, z_down=2,
loss_type="bce", optimizer="sgd", lr=0.001, n_classes=1):
"""Create 3D Residual_U-Net.
Parameters
----------
image_shape : 3D tuple
Dimensions of the input image.
activation : str, optional
Keras available activation type.
k_init : str, optional
Keras available kernel initializer type.
drop_values : array of floats, optional
Dropout value to be fixed.
batch_norm : bool, optional
Use batch normalization.
feature_maps : array of ints, optional
Feature maps to use on each level. Must have the same length as the
``depth+1``.
depth : int, optional
Depth of the network.
z_down : int, optional
Downsampling used in z dimension. Set it to 1 if the dataset is not
isotropic.
loss_type : str, optional
Loss type to use, three type available: ``bce`` (Binary Cross Entropy),
``w_bce`` (Weighted BCE, based on weigth maps) and ``w_bce_dice``
(Weighted loss: ``weight1*BCE + weight2*Dice``).
optimizer : str, optional
Optimizer used to minimize the loss function. Posible options: ``sgd``
or ``adam``.
lr : float, optional
Learning rate value.
n_classes: int, optional
Number of classes.
Returns
-------
Model : Keras model
Model containing the U-Net.
Calling this function with its default parameters returns the following
network:
.. image:: img/resunet_3d.png
:width: 100%
:align: center
Where each green layer represents a residual block as the following:
.. image:: img/res_block.png
:width: 45%
:align: center
Images created with `PlotNeuralNet <https://github.com/HarisIqbal88/PlotNeuralNet>`_.
"""
if len(feature_maps) != depth+1:
raise ValueError("feature_maps dimension must be equal depth+1")
if len(drop_values) != depth+1:
raise ValueError("'drop_values' dimension must be equal depth+1")
fm = feature_maps[::-1]
inputs = Input(image_shape)
x = level_block(inputs, depth, fm, 3, activation, k_init, drop_values,
batch_norm, True)
outputs = Conv3D(n_classes, (1, 1, 1), activation='sigmoid') (x)
model = Model(inputs=[inputs], outputs=[outputs])
# Select the optimizer
if optimizer == "sgd":
opt = tf.keras.optimizers.SGD(
lr=lr, momentum=0.99, decay=0.0, nesterov=False)
elif optimizer == "adam":
opt = tf.keras.optimizers.Adam(
lr=lr, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0,
amsgrad=False)
else:
raise ValueError("Error: optimizer value must be 'sgd' or 'adam'")
# Compile the model
if loss_type == "bce":
if n_classes > 1:
model.compile(optimizer=opt, loss='categorical_crossentropy',
metrics=[jaccard_index_softmax])
else:
model.compile(optimizer=opt, loss='binary_crossentropy',
metrics=[jaccard_index])
elif loss_type == "w_bce":
model.compile(optimizer=opt, loss=binary_crossentropy_weighted(weights),
metrics=[jaccard_index])
elif loss_type == "w_bce_dice":
model.compile(optimizer=opt,
loss=weighted_bce_dice_loss(w_dice=0.66, w_bce=0.33),
metrics=[jaccard_index])
else:
raise ValueError("'loss_type' must be 'bce', 'w_bce' or 'w_bce_dice'")
return model
def MudNet(input_shapes, output_classes, regularizer, dropout_rate,
learning_rate):
# Input layers
input_mri = Input(shape=input_shapes['mri'], name='mri_features')
input_clinical = Input(shape=input_shapes['clinical'],
name='clinical_features')
# Convolutional Layers (MRI)
x = Conv_Layer(24,
kernel_size=(11, 13, 11),
kernel_regularizer=l2(regularizer['mri']),
dropout_rate=dropout_rate['mri'],
strides=4)(input_mri)
x = Conv_Layer(48,
kernel_size=(3, 4, 3),
kernel_regularizer=l2(regularizer['mri']),
dropout_rate=dropout_rate['mri'])(x)
# Pre-activation and normalisation residual
residual = Conv3D(96,
kernel_size=(3, 4, 3),
kernel_regularizer=l2(regularizer['mri']),
padding='same')(x)
x = BatchNormalization()(residual)
x = ELU()(x)
x = Dropout(dropout_rate['mri'])(x)
x = Conv_ResidualLayer(96,
kernel_size=(3, 4, 3),
kernel_regularizer=l2(regularizer['mri']),
dropout_rate=dropout_rate['mri'])(x)
x = Conv_ResidualLayer(96,
kernel_size=(3, 4, 3),
kernel_regularizer=l2(regularizer['mri']),
dropout_rate=dropout_rate['mri'])(x)
x = Conv_ResidualLayer(96,
kernel_size=(3, 4, 3),
kernel_regularizer=l2(regularizer['mri']),
dropout_rate=dropout_rate['mri'],
residual=residual)(x)
x = Conv_Layer(24,
kernel_size=(3, 4, 3),
kernel_regularizer=l2(regularizer['mri']),
dropout_rate=dropout_rate['mri'])(x)
x = Conv_Layer(8,
kernel_size=(3, 4, 3),
kernel_regularizer=l2(regularizer['mri']),
dropout_rate=dropout_rate['mri'])(x)
# Flattened layer
mri_dense = Flatten()(x)
# Dense layers (Clinical)
x = Dense_Layer(20,
kernel_regularizer=l2(regularizer['clinical']),
dropout_rate=dropout_rate['clinical'])(input_clinical)
x = Dense_Layer(20,
kernel_regularizer=l2(regularizer['clinical']),
dropout_rate=dropout_rate['clinical'])(x)
clinical_dense = Dense_Layer(10,
kernel_regularizer=l2(
regularizer['clinical']),
dropout_rate=dropout_rate['clinical'])(x)
# Mixed layer
mixed_layer = concatenate([mri_dense, clinical_dense])
output = Dense_Layer(4,
kernel_regularizer=l2(regularizer['fc']),
dropout_rate=dropout_rate['clinical'])(mixed_layer)
# Output layers
output_conversion = Dense(output_classes['conversion'],
activation='sigmoid',
name='Conversion')(output)
output_risk = Dense(output_classes['risk'],
activation='softmax',
name='Risk')(output)
# Model compilation
model = Model(inputs=[input_mri, input_clinical],
outputs=[output_conversion, output_risk],
name="MudNet")
optimizer = Adam(learning_rate)
model.compile(loss={
'Conversion': binary_crossentropy,
'Risk': categorical_crossentropy
},
optimizer=optimizer,
metrics={
'Conversion': [binary_accuracy,
AUC(), Recall()],
'Risk': [categorical_accuracy,
AUC(), Recall()]
})
return model
def get_3d_unet_9layers(input_shape):
img_input = Input(input_shape)
conv1 = conv_block_simple_3D(img_input, 32, "conv1_1")
conv1 = conv_block_simple_3D(conv1, 32, "conv1_2")
pool1 = MaxPooling3D((2, 2, 2),
strides=(2, 2, 2),
padding="same",
name="pool1",
data_format='channels_first')(conv1)
conv2 = conv_block_simple_3D(pool1, 64, "conv2_1")
conv2 = conv_block_simple_3D(conv2, 64, "conv2_2")
pool2 = MaxPooling3D((2, 2, 2),
strides=(2, 2, 2),
padding="same",
name="pool2",
data_format='channels_first')(conv2)
conv3 = conv_block_simple_3D(pool2, 128, "conv3_1")
conv3 = conv_block_simple_3D(conv3, 128, "conv3_2")
pool3 = MaxPooling3D((2, 2, 2),
strides=(2, 2, 2),
padding="same",
name="pool3",
data_format='channels_first')(conv3)
conv4 = conv_block_simple_3D(pool3, 256, "conv4_1")
conv4 = conv_block_simple_3D(conv4, 256, "conv4_2")
pool4 = MaxPooling3D((2, 2, 2),
strides=(2, 2, 2),
padding="same",
name="pool4",
data_format='channels_first')(conv4)
conv5 = conv_block_simple_3D(pool4, 512, "conv5_1")
conv5 = conv_block_simple_3D(conv5, 512, "conv5_2")
conv5 = conv_block_simple_3D(conv5, 512, "conv5_3")
up6 = concatenate(
[UpSampling3D(data_format='channels_first')(conv5), conv4], axis=1)
conv6 = conv_block_simple_3D(up6, 256, "conv6_1")
conv6 = conv_block_simple_3D(conv6, 256, "conv6_2")
up7 = concatenate(
[UpSampling3D(data_format='channels_first')(conv6), conv3], axis=1)
conv7 = conv_block_simple_3D(up7, 128, "conv7_1")
conv7 = conv_block_simple_3D(conv7, 128, "conv7_2")
up8 = concatenate(
[UpSampling3D(data_format='channels_first')(conv7), conv2], axis=1)
conv8 = conv_block_simple_3D(up8, 64, "conv8_1")
conv8 = conv_block_simple_3D(conv8, 64, "conv8_2")
up9 = concatenate(
[UpSampling3D(data_format='channels_first')(conv8), conv1], axis=1)
conv9 = conv_block_simple_3D(up9, 32, "conv9_1")
conv9 = conv_block_simple_3D(conv9, 32, "conv9_2")
conv9 = SpatialDropout3D(rate=0.1, data_format='channels_first')(conv9)
prediction = Conv3D(1, (1, 1, 1),
activation="sigmoid",
name="prediction",
data_format='channels_first')(conv9)
model = Model(img_input, prediction)
return model
def resnet_3d_cnn(self, voxel_dim, deviation_channels, w_val=0):
import numpy as np
import tensorflow.keras.backend as K
from tensorflow.keras.models import Model
from tensorflow.keras import regularizers
from tensorflow.keras.layers import Conv3D, MaxPooling3D, Add, BatchNormalization, Input, LeakyReLU, Activation, Lambda, Concatenate, Flatten, Dense, UpSampling3D, GlobalAveragePooling3D
if (w_val == 0):
w_val = np.zeros(self.output_dimension)
w_val[:] = 1 / self.output_dimension
def weighted_mse(val):
def loss(yTrue, yPred):
#val = np.array([0.1,0.1,0.1,0.1,0.1])
w_var = K.variable(value=val,
dtype='float32',
name='weight_vec')
#weight_vec = K.ones_like(yTrue[0,:]) #a simple vector with ones shaped as (60,)
#idx = K.cumsum(ones) #similar to a 'range(1,61)'
return K.mean((w_var) * K.square(yTrue - yPred))
return loss
input_size = (voxel_dim, voxel_dim, voxel_dim, deviation_channels)
inputs = Input(input_size)
x = inputs
y = Conv3D(32,
kernel_size=(4, 4, 4),
strides=(2, 2, 2),
name="conv_block_1")(x)
res1 = y
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
name="conv_block_2")(y)
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
name="conv_block_3")(y)
y = Add()([res1, y])
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(2, 2, 2),
name="conv_block_4")(y)
res2 = y
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
name="conv_block_5")(y)
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
name="conv_block_6")(y)
y = Add()([res2, y])
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(2, 2, 2),
name="conv_block_7")(y)
res3 = y
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
name="conv_block_8")(y)
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
name="conv_block_9")(y)
y = Add()([res3, y])
y = LeakyReLU()(y)
y = Flatten()(y)
y = Dense(128,
kernel_regularizer=regularizers.l2(self.regularizer_coeff),
activation='relu')(y)
y = Dense(64,
kernel_regularizer=regularizers.l2(self.regularizer_coeff),
activation='relu')(y)
output = Dense(self.output_dimension)(y)
model = Model(inputs, outputs=output, name='Res_3D_CNN')
model.compile(loss=self.loss_function,
optimizer=self.optimizer,
metrics=['mae'])
#print(model.summary())
return model
def f(input):
conv = Conv3D(filters=filters, kernel_size=kernel_size,
strides=strides, kernel_initializer=kernel_initializer,
padding=padding,
kernel_regularizer=kernel_regularizer)(input)
return _bn_relu(conv)
def f(x):
if bottleneck:
y = Conv3D(
output_features,
bottleneck_conv_kernel,
strides=stride,
use_bias=use_bias,
name=f"resunit{resnet_unit_label}_block{block_id}_conv_a",
kernel_initializer=kernel_initializer,
)(x)
else:
y = ZeroPadding3D(
padding=1,
name=f"resunit{resnet_unit_label}_block{block_id}_pad_a",
)(x)
y = Conv3D(
output_features,
conv_kernel,
strides=stride,
use_bias=use_bias,
name=f"resunit{resnet_unit_label}_block{block_id}_conv_a",
kernel_initializer=kernel_initializer,
)(y)
y = BatchNormalization(
axis=axis,
epsilon=bn_epsilon,
name=f"resunit{resnet_unit_label}_block{block_id}_bn_a",
)(y)
y = Activation(
activation,
name=f"resunit{resnet_unit_label}_block{block_id}_activation_a",
)(y)
y = ZeroPadding3D(
padding=1, name=f"resunit{resnet_unit_label}_block{block_id}_pad_b"
)(y)
y = Conv3D(
output_features,
conv_kernel,
use_bias=use_bias,
name=f"resunit{resnet_unit_label}_block{block_id}_conv_b",
kernel_initializer=kernel_initializer,
)(y)
y = BatchNormalization(
axis=axis,
epsilon=bn_epsilon,
name=f"resunit{resnet_unit_label}_block{block_id}_bn_b",
)(y)
if bottleneck:
y = Activation(
activation,
name=f"resunit{resnet_unit_label}_block{block_id}_activation_b",
)(y)
y = Conv3D(
output_features * 4,
bottleneck_conv_kernel,
use_bias=use_bias,
name=f"resunit{resnet_unit_label}_block{block_id}_conv_c",
kernel_initializer=kernel_initializer,
)(y)
y = BatchNormalization(
axis=axis,
epsilon=bn_epsilon,
name=f"resunit{resnet_unit_label}_block{block_id}_bn_c",
)(y)
identity_shortcut = get_shortcut(
x,
resnet_unit_label,
block_id,
output_features * 4,
stride,
axis=axis,
)
else:
identity_shortcut = get_shortcut(
x,
resnet_unit_label,
block_id,
output_features,
stride,
axis=axis,
)
y = Add(name=f"resunit{resnet_unit_label}_block{block_id}_add")(
[y, identity_shortcut]
)
y = Activation(
activation,
name=f"resunit{resnet_unit_label}_block{block_id}_activation_c",
)(y)
return y
def f(input):
activation = _bn_relu(input)
return Conv3D(filters=filters, kernel_size=kernel_size,
strides=strides, kernel_initializer=kernel_initializer,
padding=padding,
kernel_regularizer=kernel_regularizer)(activation)
def encode_block(inp_layer,
channels,
t_downsmp_layer=4,
downsample=False,
fixed_dropout=0.1):
"""Encode block defined in Cheng et al.
Parameters
----------
inp_layer : Keras layer
Input layer.
channels : int, optional
Feature maps to define in Conv layers.
t_downsmp_layer : int, optional
``t`` value defined in the paper for the proposed stochastic
downsampling layer.
downsample : bool, optional
To make a downsampling. Blue blocks in the encoding part.
fixed_dropout : float, optional
Dropout value.
Returns
-------
out : Keras layer
Last layer of the block.
"""
if downsample == True:
shortcut_padded = StochasticDownsampling3D()(inp_layer,
t_downsmp_layer)
shortcut_padded = Conv3D(channels, (1, 1, 1),
activation=None,
kernel_regularizer=l2(0.01))(shortcut_padded)
else:
shortcut_padded = Lambda(pad_depth,
arguments={'desired_channels':
channels})(inp_layer)
x = BatchNormalization()(inp_layer)
x = PReLU(shared_axes=[1, 2, 3])(x)
if downsample == True:
r = 1 if channels % 3 > 0 else 0
c1 = Conv3D(int(channels / 3) + r, (1, 1, 3),
activation=None,
strides=(2, 2, 2),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01))(x)
r = 1 if channels % 3 > 1 else 0
c2 = Conv3D(int(channels / 3) + r, (1, 3, 1),
activation=None,
strides=(2, 2, 2),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01))(x)
c3 = Conv3D(int(channels / 3), (3, 1, 1),
activation=None,
strides=(2, 2, 2),
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01))(x)
x = concatenate([c1, c2, c3])
else:
r = 1 if channels % 3 > 0 else 0
c1 = Conv3D(int(channels / 3) + r, (1, 1, 3),
activation=None,
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01))(x)
r = 1 if channels % 3 > 1 else 0
c2 = Conv3D(int(channels / 3) + r, (1, 3, 1),
activation=None,
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01))(x)
c3 = Conv3D(int(channels / 3), (3, 1, 1),
activation=None,
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01))(x)
x = concatenate([c1, c2, c3])
x = Dropout(fixed_dropout)(x)
x = BatchNormalization()(x)
x = PReLU(shared_axes=[1, 2, 3])(x)
r = 1 if channels % 3 > 0 else 0
c1 = Conv3D(int(channels / 3) + r, (1, 1, 3),
activation=None,
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01))(x)
r = 1 if channels % 3 > 1 else 0
c2 = Conv3D(int(channels / 3) + r, (1, 3, 1),
activation=None,
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01))(x)
c3 = Conv3D(int(channels / 3), (3, 1, 1),
activation=None,
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01))(x)
x = concatenate([c1, c2, c3])
x = Add()([shortcut_padded, x])
return x
def __init__(self):
gpus = tf.config.experimental.list_physical_devices('GPU')
#gpus = None
if gpus:
# Restrict TensorFlow to only allocate 1GB of memory on the first GPU
try:
tf.config.experimental.set_virtual_device_configuration(
gpus[0],
[tf.config.experimental.VirtualDeviceConfiguration(memory_limit=4096)])
logical_gpus = tf.config.experimental.list_logical_devices('GPU')
print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
except RuntimeError as e:
# Virtual devices must be set before GPUs have been initialized
print(e)
self.single_img_length = 200
self.single_input = keras.Input(shape=(2*self.single_img_length, 10 * self.single_img_length, 3), \
name="Input")
self.acce_input = self.single_input[:, 0:self.single_img_length, :, :]
self.gyro_input = self.single_input[:, self.single_img_length:, :, :]
self.acce_input = Reshape((10, self.single_img_length, self.single_img_length, 3), name="Input_acce")(
self.acce_input)
self.gyro_input = Reshape((10, self.single_img_length, self.single_img_length, 3), name="Input_gyro")(
self.gyro_input)
acc_input_dim = self.acce_input.get_shape()
print(acc_input_dim)
gyro_input_dim = self.gyro_input.get_shape()
print(gyro_input_dim)
self.conv1a = Conv3D(64, (1, 5, 5), (1, 3, 3),
name="conv_acce_1",
kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
)(self.acce_input)
# self.conv1a = self.acce_input
# self.conv1a = BatchNormalization(name="conv_acce_1_batchnorm")(self.conv1a)
self.conv1a = Activation("relu", name="conv_acce_1_relu")(self.conv1a)
self.conv1a = Dropout(DROPOUT_RATIO, noise_shape=[BATCH_SIZE, 1, 1, 1, self.conv1a.shape[-1]], name="conv_acce_1_dropout")(
self.conv1a)
self.conv1a = AveragePooling3D((1, 2, 2),name="conv_acce_1_pool")(self.conv1a)
acc_conv1a_dim = self.conv1a.get_shape()
print(acc_conv1a_dim)
self.conv2a = Conv3D(64, (1, 3, 3), (1, 1, 1),
name="conv_acce_2",
padding="SAME",
kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
)(self.conv1a)
# self.conv2a = BatchNormalization(name="conv_acce_2_batchnorm")(self.conv2a)
self.conv2a = Activation("relu", name="conv_acce_2_relu")(self.conv2a)
self.conv2a = Dropout(DROPOUT_RATIO, noise_shape=[BATCH_SIZE, 1, 1, 1, self.conv1a.shape[-1]], name="conv_acce_2_dropout")(
self.conv2a)
self.conv2a = AveragePooling3D((1, 2, 2), name="conv_acce_2_pool")(self.conv1a)
# self.conv3a = Conv3D(64, (1, 3, 3), (1, 2, 2),
# name="conv_acce_3",
# kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
# )(self.conv2a)
# # self.conv3a = BatchNormalization(name="conv_acce_3_batchnorm")(self.conv3a)
# self.conv3a = Activation("relu", name="conv_acce_3_relu")(self.conv3a)
# self.conv3a = Dropout(DROPOUT_RATIO, noise_shape=[BATCH_SIZE, 1, 1, 1, 64], name="conv_acce_3_dropout")(
# self.conv3a)
acc_conv2a_dim = self.conv2a.get_shape()
print(acc_conv2a_dim)
self.acce_output = Reshape((10, 1, 16*16, self.conv1a.shape[-1]), name="output_acce")(self.conv2a)
# self.acce_output = self.conv3a
# self.acce_output = AveragePooling3D((1,3,3),(1,2,2))(self.conv3a)
# self.acce_output = Reshape((10,1,10*10,64),name="output_acce")(self.acce_output)
# **********************************************************************************************
acc_output_dim = self.acce_output.get_shape()
print(acc_output_dim)
self.conv1g = Conv3D(64, (1, 5, 5), (1, 3, 3),
name="conv_gyro_1",
kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
)(self.gyro_input)
# self.conv1g = self.gyro_input
# self.conv1g = BatchNormalization(name="conv_gyro_1_batchnorm")(self.conv1g)
self.conv1g = Activation("relu", name="conv_gyro_1_relu")(self.conv1g)
self.conv1g = Dropout(DROPOUT_RATIO, noise_shape=[BATCH_SIZE, 1, 1, 1, self.conv1g.shape[-1]], name="conv_gyro_1_dropout")(
self.conv1g)
self.conv1g = AveragePooling3D((1, 2, 2), name="conv_gyro_1_pool")(self.conv1g)
gyro_conv1g_dim = self.conv1g.get_shape()
print(gyro_conv1g_dim)
self.conv2g = Conv3D(64, (1, 3, 3), (1, 1, 1),
name="conv_gyro_2",
padding="SAME",
kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
)(self.conv1g)
# self.conv2g = BatchNormalization(name="conv_gyro_2_batchnorm")(self.conv2g)
self.conv2g = Activation("relu", name="conv_gyro_2_relu")(self.conv2g)
self.conv2g = Dropout(DROPOUT_RATIO, noise_shape=[BATCH_SIZE, 1, 1, 1, self.conv2g.shape[-1]],
name="conv_gyro_2_dropout")(self.conv2g)
self.conv2g = AveragePooling3D((1, 2, 2), name="conv_gyro_2_pool")(self.conv1g)
# self.conv3g = Conv3D(64, (1, 3, 3), (1, 2, 2),
# name="conv_gyro_3",
# kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
# )(self.conv2g)
# # self.conv3g = BatchNormalization(name="conv_gyro_3_batchnorm")(self.conv3g)
# self.conv3g = Activation("relu", name="conv_gyro_3_relu")(self.conv3g)
# self.conv3g = Dropout(DROPOUT_RATIO, noise_shape=[BATCH_SIZE, 1, 1, 1, 64], name="conv_gyro_3_dropout")(
# self.conv3g)
gyro_conv2g_dim = self.conv2g.get_shape()
print(gyro_conv2g_dim)
self.gyro_output = Reshape((10, 1, 16*16, self.conv1g.shape[-1]), name="output_gyro")(self.conv2g)
# self.gyro_output = self.conv3g
# self.gyro_output = AveragePooling3D((1,3,3),(1,2,2))(self.conv3g)
# self.gyro_output = Reshape((10,1,10*10,64),name="output_gyro")(self.gyro_output)
# ***********************************************************************************************************************
gyro_output_dim = self.gyro_output.get_shape()
print(gyro_output_dim)
#============================no attention===========================================
self.merge_noattention = concatenate([self.acce_output,self.gyro_output],axis = -3, name = "input_merge")
self.merge_input = self.merge_noattention
#==============================attention====================================================
#self.merge_input = concatenate([self.acce_output, self.gyro_output], axis=-3, name="Input_merge")
#self.merge_attention_input = Reshape((10,2,self.merge_input.shape[-2]*self.merge_input.shape[-1]))(self.merge_input)
#self.merge_attention = tf.matmul(self.merge_attention_input,self.merge_attention_input,transpose_b=True)
#self.merge_attention = tf.matmul(tf.ones((1,2),dtype=tf.float32),self.merge_attention)
#softmax_layer = Softmax(axis = -1)
#self.merge_attention = softmax_layer(self.merge_attention)
#merge_attention_dim = self.merge_attention.get_shape()
#print("merge_attention_dim")
#print(merge_attention_dim)
#self.merge_attention_output = tf.matmul(self.merge_attention,self.merge_attention_input)
#self.merge_attention_output = Reshape((10,16,16,64))(self.merge_attention_output)
#merge_attention_output_dim = self.merge_attention_output.get_shape()
#print("merge_attention_output_dim")
#print(merge_attention_output_dim)
#self.merge_input = self.merge_attention_output
#===========================================================================================
self.conv1 = Conv3D(64, kernel_size=(1,2,5),
name='conv_merge_1',
strides=( 1, 1,1),
# padding='SAME',
kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
)(self.merge_input)
merge_conv1_dim = self.conv1.get_shape()
print("merge_conv1_dim")
print(merge_conv1_dim)
# self.conv1 = BatchNormalization(name="conv_merge_1_batchnorm")(self.conv1)
self.conv1 = Activation("relu", name="conv_merge_1_relu")(self.conv1)
self.conv1 = Dropout(0.2, noise_shape=[BATCH_SIZE, 1, 1, 1, self.conv1.shape[-1]],
name="conv_merge_1_dropout")(self.conv1)
self.conv1 = AveragePooling3D((1, 1,3))(self.conv1)
# self.conv2 = Conv3D(64, kernel_size=(1, 1, 5),
# name="conv_merge_2",
# strides=(1, 1, 3),
# # padding='SAME',
# kernel_regularizer=keras.regularizers.l2(REGULARIZER_RATE)
# )(self.conv1)
# # self.conv2 = BatchNormalization(name="conv_merge_2_batchnorm")(self.conv2)
# self.conv2 = Activation("relu", name="conv_merge_2_relu")(self.conv2)
# self.conv2 = Dropout(DROPOUT_RATIO, noise_shape=[BATCH_SIZE, 1, 1, 1, 64], name="conv_merge_2_dropout")(
# self.conv2)
self.conv_output = self.conv1
self.rnn_input = Reshape((10, self.conv_output.shape[-1] * 84), name="Output_merge")(self.conv_output)
self.rnn = LSTM(120, return_sequences=True, name="RNN_1")(self.rnn_input)
self.sum_rnn_out = tf.reduce_sum(self.rnn, axis=1, keep_dims=False)
self.rnn = LSTM(20, name="RNN_2")(self.rnn)
self.rnn_output = Dense(6, 'softmax', name="Softmax")(self.rnn)
self.model = keras.Model(
inputs=self.single_input,
outputs=self.rnn_output)
curr_time = f'{datetime.now():%H-%M-%S%z_%m%d%Y}'
logger_path = "/pylon5/cc5614p/deopha32/Saved_Models/adhd-fmri-history_cv{num}_{time}.csv".format(
num=file_num, time=curr_time)
csv_logger = CSVLogger(logger_path, append=True)
callbacks = [csv_logger]
# ============================ MODEL ARCHITECTURE ============================
with tf.device('/gpu:0'):
cnn_lstm_model = Sequential()
cnn_lstm_model.add(
TimeDistributed(Conv3D(filters=64,
kernel_size=(3, 3, 3),
activation='relu'),
input_shape=input_shape,
name="Input_Conv_Layer"))
cnn_lstm_model.add(
TimeDistributed(MaxPool3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
padding='valid'),
name="Pool_Layer_1"))
cnn_lstm_model.add(TimeDistributed(Flatten(), name="Flatten_Layer"))
with tf.device('/cpu:0'):
cnn_lstm_model.add(
model.add(
ConvLSTM2D(filters=64,
kernel_size=(3, 3),
padding='same',
return_sequences=True,
use_bias=True))
model.add(BatchNormalization())
model.add(
ConvLSTM2D(filters=64,
kernel_size=(3, 3),
padding='same',
return_sequences=True,
use_bias=True))
model.add(BatchNormalization())
model.add(
Conv3D(filters=1,
kernel_size=(3, 3, 3),
padding='same',
data_format='channels_last'))
MODEL_DIR = os.getcwd()
version = 1
export_path = os.path.join(MODEL_DIR, str(version))
print('export_path= {}\n'.format(export_path))
keras.models.save_model(model, export_path, save_format='tf')
print('\nSaved model:')
def build_model(self, inputs):
""" Builds the model with all the pieces put together:
1) General Conv + activation
2) n Residual Block (default)
3) General Conv + batch norm + skip connection
4) Upsample Block
5) Final Convulation + activation to generate HR
Args:
inputs: Keras.layers.input input shape
Returns:
a tf.keras.Model, the built model
"""
model = Conv2D(32,
3,
strides=1,
padding='same',
kernel_initializer='he_normal')(inputs)
model = LeakyReLU(alpha=0.2)(model)
model = Conv2D(32,
3,
strides=1,
padding='same',
kernel_initializer='he_normal')(model)
model = LeakyReLU(alpha=0.2)(model)
model = Conv2D(64,
3,
strides=1,
padding='same',
kernel_initializer='he_normal')(model)
model = LeakyReLU(alpha=0.2)(model)
model = Conv2D(64,
3,
strides=1,
padding='same',
kernel_initializer='he_normal')(model)
skip_connection = model
# residual attentions
for i in range(self.number_residual_block):
model = residual_attention_block(model, 3, 128, 1, self.batch_norm)
model = Conv3D(128, kernel_size=3, strides=1, padding="SAME")
model = LeakyReLU(alpha=0.2)(model)
# upscale
model = up_sample_block(model, 3, 128, 1, size=3, skip=None)
model = Conv3D(128, kernel_size=3, strides=1, padding="SAME")
model = Conv3D(64, kernel_size=3, strides=1, padding="SAME")
LeakyReLU(alpha=0.2)(model)
model = Conv2D(1,
kernel_size=3,
strides=1,
padding="same",
kernel_initializer='he_normal')(model)
return tf.keras.Model(inputs=inputs, outputs=model)
def C3D_model(input_shape, nb_classes):
inp_3d = Input(shape=(input_shape), name='3d_input')
x = Conv3D(64,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
activation='relu')(inp_3d)
x = MaxPooling3D(pool_size=(1, 2, 2), strides=(1, 2, 2), padding='same')(x)
print(x.shape)
x = Conv3D(128,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
activation='relu')(x)
x = MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), padding='same')(x)
print(x.shape)
x = Conv3D(256,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
activation='relu')(x)
x = Conv3D(256,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
activation='relu')(x)
x = MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), padding='same')(x)
print(x.shape)
x = Conv3D(512,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
activation='relu')(x)
x = Conv3D(512,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
activation='relu')(x)
x = MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), padding='same')(x)
print(x.shape)
x = Conv3D(512,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
activation='relu')(x)
x = Conv3D(512,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
activation='relu')(x)
x = MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2), padding='same')(x)
print(x.shape)
x = Flatten()(x)
x = Dense(1024, activation='relu')(x)
x = Dropout(0.5)(x)
x = Dense(512, activation='relu')(x)
x = Dropout(0.5)(x)
x = Dense(nb_classes)(x)
x = Activation('softmax')(x)
model = Model(inp_3d, x)
return model
def Unet3D_Model(inputs_shape):
# 定义输入
input_layer = Input(inputs_shape)
# Conv3D-->BatchNormalization-->Activation * 2
x1 = Conv3D(filters=32, kernel_size=(3,3,3), padding='same', strides=(1,1,1))(input_layer)
x1 = BatchNormalization()(x1)
# x1 = PReLU(x1)
x1 = Activation('relu')(x1)
x1 = Conv3D(filters=64, kernel_size=(3,3,3), padding='same', strides=(1,1,1))(x1)
x1 = BatchNormalization()(x1)
x1 = Activation('relu')(x1)
# M + CBA * 2 (缩小特征图尺度)
x2 = MaxPooling3D(pool_size=(2,2,2))(x1)
x2 = Conv3D(filters=64, kernel_size=(3,3,3), padding='same', strides=(1,1,1))(x2)
x2 = BatchNormalization()(x2)
x2 = Activation('relu')(x2)
x2 = Conv3D(filters=128, kernel_size=(3,3,3), padding='same', strides=(1,1,1))(x2)
x2 = BatchNormalization()(x2)
x2 = Activation('relu')(x2)
# M + CBA * 2 (缩小特征图尺度)
x3 = MaxPooling3D(pool_size=(2,2,2))(x2)
x3 = Conv3D(filters=128, kernel_size=(3,3,3), padding='same', strides=(1,1,1))(x3)
x3 = BatchNormalization()(x3)
x3 = Activation('relu')(x3)
x3 = Conv3D(filters=256, kernel_size=(3,3,3), padding='same', strides=(1,1,1))(x3)
x3 = BatchNormalization()(x3)
x3 = Activation('relu')(x3)
# M + CBA * 2 (缩小特征图尺度)
x4 = MaxPooling3D(pool_size=(2,2,2))(x3)
x4 = Conv3D(filters=256, kernel_size=(3,3,3), padding='same', strides=(1,1,1))(x4)
x4 = BatchNormalization()(x4)
x4 = Activation('relu')(x4)
x4 = Conv3D(filters=512, kernel_size=(3,3,3), padding='same', strides=(1,1,1))(x4)
x4 = BatchNormalization()(x4)
x4 = Activation('relu')(x4)
# CT + ca + CBA * 2 (增大特征图尺度+跳跃式链接)
x5 = Conv3DTranspose(filters=512, kernel_size=(2,2,2), padding='same', strides=(2,2,2))(x4)
x5 = concatenate([x5, x3], axis=1)
x5 = Conv3D(filters=256, kernel_size=(3,3,3), padding='same', strides=(1,1,1))(x5)
x5 = BatchNormalization()(x5)
x5 = Activation('relu')(x5)
x5 = Conv3D(filters=256, kernel_size=(3,3,3), padding='same', strides=(1,1,1))(x5)
x5 = BatchNormalization()(x5)
x5 = Activation('relu')(x5)
# CT + ca + CBA * 2 (增大特征图尺度+跳跃式链接)
x6 = Conv3DTranspose(filters=256, kernel_size=(2,2,2), padding='same', strides=(2,2,2))(x5)
x6 = concatenate([x6, x2], axis=1)
x6 = Conv3D(filters=128, kernel_size=(3,3,3), padding='same', strides=(1,1,1))(x6)
x6 = BatchNormalization()(x6)
x6 = Activation('relu')(x6)
x6 = Conv3D(filters=128, kernel_size=(3,3,3), padding='same', strides=(1,1,1))(x6)
x6 = BatchNormalization()(x6)
x6 = Activation('relu')(x6)
# CT + ca + CBA * 2 (增大特征图尺度+跳跃式链接)
x7 = Conv3DTranspose(filters=128, kernel_size=(2,2,2), padding='same', strides=(2,2,2))(x6)
x7 = concatenate([x7, x1], axis=1)
x7 = Conv3D(filters=64, kernel_size=(3,3,3), padding='same', strides=(1,1,1))(x7)
x7 = BatchNormalization()(x7)
x7 = Activation('relu')(x7)
x7 = Conv3D(filters=64, kernel_size=(3,3,3), padding='same', strides=(1,1,1))(x7)
x7 = BatchNormalization()(x7)
x7 = Activation('relu')(x7)
#
x8 = Conv3D(filters=1, kernel_size=(1,1,1), padding='same', strides=(1,1,1))(x7)
x8 = Activation('sigmoid')(x8)
model = Model(inputs=input_layer, outputs=x8)
model.compile(optimizer=Adam(lr=config['initial_learning_rate']), loss=weighted_dice_coefficient_loss, metrics=[weighted_dice_coefficient])
return model
def __merge_temporal_features(feature,
mode='conv',
feature_size=256,
frames_per_batch=1):
"""Merges feature with its temporal residual through addition.
Input feature (x) --> Temporal convolution* --> Residual feature (x')
*Type of temporal convolution specified by ``mode``.
Output: ``y = x + x'``
Args:
feature (tensorflow.keras.layers.Layer): Input layer
mode (str): Mode of temporal convolution. One of
``{'conv','lstm','gru', None}``.
feature_size (int): Length of convolutional kernel
frames_per_batch (int): Size of z axis in generated batches.
If equal to 1, assumes 2D data.
Raises:
ValueError: ``mode`` not 'conv', 'lstm', 'gru' or ``None``
Returns:
tensorflow.keras.layers.Layer: Input feature merged with its residual
from a temporal convolution. If mode is ``None``,
the output is exactly the input.
"""
# Check inputs to mode
acceptable_modes = {'conv', 'lstm', 'gru', None}
if mode is not None:
mode = str(mode).lower()
if mode not in acceptable_modes:
raise ValueError('Mode {} not supported. Please choose '
'from {}.'.format(mode, str(acceptable_modes)))
f_name = str(feature.name)[:2]
if mode == 'conv':
x = Conv3D(
feature_size,
(frames_per_batch, 3, 3),
strides=(1, 1, 1),
padding='same',
name='conv3D_mtf_{}'.format(f_name),
)(feature)
x = BatchNormalization(axis=-1, name='bnorm_mtf_{}'.format(f_name))(x)
x = Activation('relu', name='acti_mtf_{}'.format(f_name))(x)
elif mode == 'lstm':
x = ConvLSTM2D(feature_size, (3, 3),
padding='same',
activation='relu',
return_sequences=True,
name='convLSTM_mtf_{}'.format(f_name))(feature)
elif mode == 'gru':
x = ConvGRU2D(feature_size, (3, 3),
padding='same',
activation='relu',
return_sequences=True,
name='convGRU_mtf_{}'.format(f_name))(feature)
else:
x = feature
temporal_feature = x
return temporal_feature
