def __init__(self, shape):
self.re_rate = 0.9
self.inputs = layers.Input(shape=shape)
self.f_block = layers.Conv3D(4, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.inputs)
self.bn = layers.BatchNormalization()(self.f_block)
self.mp1 = layers.MaxPooling3D((2, 2, 2))(self.bn)
self.f_block1 = layers.Conv3D(8, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.mp1)
self.bn = layers.BatchNormalization()(self.f_block1)
self.mp2 = layers.MaxPooling3D((2, 2, 2))(self.bn)
self.f_block2 = layers.Conv3D(8, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.mp2)
self.f_block2 = layers.BatchNormalization()(self.f_block2)
self.b_back2 = layers.Conv3D(8, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.f_block2)
self.b_back2 = layers.BatchNormalization()(self.b_back2)
self.b_back2 = layers.Conv3D(
8, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(layers.UpSampling3D((2, 2, 2))(self.f_block2))
self.b_back2 = layers.BatchNormalization()(self.b_back2)
self.cat2 = layers.concatenate([self.f_block1, self.b_back2])
self.bn = layers.BatchNormalization()(self.cat2)
self.b_back1 = layers.Conv3D(
8, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(layers.UpSampling3D((2, 2, 2))(self.bn))
self.b_back1 = layers.BatchNormalization()(self.b_back1)
self.gb = layers.GlobalAveragePooling3D()(self.b_back1)
self.drop = layers.Dropout(rate=0.9)(self.gb)
self.dense = layers.Dense(1, activation='sigmoid')(self.drop)
self.model = keras.Model(input=[self.inputs], output=self.dense)
def reverse_resnet_graph(layer, architecture, stage5=False):
# # Stage 5
if stage5:
layer = KL.UpSampling3D((2, 2, 2))(layer)
layer = conv_block(layer,
3, [192, 192],
stage=5,
block='arev',
strides=(1, 1, 1))
# Stage 4
#block_count = {"resnet50": 5, "resnet101": 22}[architecture]
layer = KL.UpSampling3D((2, 2, 2))(layer)
# layer = KL.Conv3D(32, (3,3,3), strides=(1, 1, 1), use_bias=True, padding='same')(layer)
# layer = KL.Activation('relu')(layer)
# for i in range(block_count):
# layer = reverse_identity_block(layer, 3, [8, 8, 32], stage=4, block=chr(98 + block_count -1 - i))
layer = conv_block(layer,
3, [96, 96],
stage=4,
block='arev',
strides=(1, 1, 1))
# Stage 3
layer = KL.UpSampling3D((2, 2, 2))(layer)
# layer = KL.Conv3D(16, (3,3,3), strides=(1, 1, 1), use_bias=True, padding='same')(layer)
# layer = KL.Activation('relu')(layer)
# layer = reverse_conv_block(layer, 3, [4, 4, 16], stage=3, block='d')
# layer = reverse_identity_block(layer, 3, [4, 4, 16], stage=3, block='c')
# layer = reverse_identity_block(layer, 3, [4, 4, 16], stage=3, block='b')
layer = conv_block(layer,
3, [48, 48],
stage=3,
block='arev',
strides=(1, 1, 1))
# Stage 2
layer = KL.UpSampling3D((2, 2, 2))(layer)
# layer = KL.Conv3D(8, (3,3,3), strides=(1, 1, 1), use_bias=True, padding='same')(layer)
# layer = KL.Activation('relu')(layer)
# layer = reverse_identity_block(layer, 3, [2, 2, 8], stage=2, block='c')
# layer = reverse_identity_block(layer, 3, [2, 2, 8], stage=2, block='b')
layer = conv_block(layer,
3, [24, 24],
stage=2,
block='arev',
strides=(1, 1, 1))
# Stage 1
layer = KL.UpSampling3D((2, 2, 2))(layer)
layer = KL.Conv3D(1, (7, 7, 7),
strides=(1, 1, 1),
name='conv_last',
use_bias=True,
padding='same')(layer)
layer = KL.Activation('relu')(layer)
return layer
def CNN():
num_channels = 1
num_mask_channels = 1
img_shape = (None, None, None, 1)
inputs = Input(shape = img_shape)
conv1 = layers.Conv3D(32, 3, padding='same')(inputs)
conv1 = layers.BatchNormalization()(conv1)
conv1 = Activation('relu')(conv1)
conv1 = layers.Conv3D(32, 3, padding='same')(conv1)
conv1 = layers.BatchNormalization()(conv1)
conv1 = Activation('relu')(conv1)
pool1 = layers.MaxPooling3D(pool_size=(2, 2, 2))(conv1)
conv2 = layers.Conv3D(64, 3, padding='same')(pool1)
conv2 = layers.BatchNormalization()(conv2)
conv2 = Activation('relu')(conv2)
conv2 = layers.Conv3D(64, 3, padding='same')(conv2)
conv2 = layers.BatchNormalization()(conv2)
conv2 = Activation('relu')(conv2)
pool2 = layers.MaxPooling3D(pool_size=(2, 2, 2))(conv2)
conv3 = layers.Conv3D(128, 3, padding='same')(pool2)
conv3 = layers.BatchNormalization()(conv3)
conv3 = Activation('relu')(conv3)
conv3 = layers.Conv3D(128, 3, padding='same')(conv3)
conv3 = layers.BatchNormalization()(conv3)
conv3 = Activation('relu')(conv3)
conv3 = layers.UpSampling3D(size=(2, 2, 2))(conv3)
up4 = layers.concatenate([conv3, conv2])
conv4 = layers.Conv3DTranspose(64, 3, padding='same')(up4)
conv4 = layers.BatchNormalization()(conv4)
conv4 = Activation('relu')(conv4)
conv4 = layers.Conv3DTranspose(64, 3, padding='same')(conv4)
conv4 = layers.BatchNormalization()(conv4)##conv ou crop
conv4 = Activation('relu')(conv4)
conv4 = layers.Conv3DTranspose(64, 1, padding='same')(conv4)
conv4 = layers.UpSampling3D(size=(2, 2, 2))(conv4)
up5 = layers.concatenate([conv4, conv1])
conv5 = layers.Conv3DTranspose(32, 3, padding='same')(up5)
conv5 = layers.BatchNormalization()(conv5)
conv5 = Activation('relu')(conv5)
conv5 = layers.Conv3DTranspose(32, 3, padding='same')(conv5)
conv5 = layers.BatchNormalization()(conv5)##conv ou crop
conv5 = Activation('relu')(conv5)
conv5 = layers.Conv3DTranspose(1, 1, padding='same', activation='relu')(conv5)
model = Model(inputs=inputs, outputs=conv5)
#model.summary()
return(model)
def __init__(self, shape):
self.re_rate = 0.9
dr = 0.9
self.inputs = layers.Input(shape=shape)
self.f_block = layers.Conv3D(4, (3, 3, 3), activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(self.inputs)
self.mp1 = layers.MaxPooling3D((2, 2, 2))(self.f_block)
self.bn1 = layers.BatchNormalization()(self.mp1)
self.f_block1 = layers.Conv3D(16, (3, 3, 3), activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(self.bn1)
self.mp2 = layers.MaxPooling3D((2, 2, 2))(self.f_block1)
self.bn2 = layers.BatchNormalization()(self.mp2)
self.f_block2 = layers.Conv3D(32, (3, 3, 3), activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(self.bn2)
self.mp3 = layers.MaxPooling3D((2, 2, 2))(self.f_block2)
self.bn3 = layers.BatchNormalization()(self.mp3)
self.f_block3 = layers.Conv3D(64, (3, 3, 3), activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(self.bn3)
self.f_block3 = layers.BatchNormalization()(self.f_block3)
self.b_back3 = layers.Conv3D(128, (3, 3, 3), activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(self.f_block3)
self.b_back3 = layers.BatchNormalization()(self.b_back3)
self.b_back2 = layers.Conv3D(64, (3, 3, 3), activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(layers.UpSampling3D((2, 2, 2))(self.b_back3))
self.b_back2 = layers.BatchNormalization()(self.b_back2)
self.b_back1 = layers.Conv3D(32, (3, 3, 3), activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(layers.UpSampling3D((2, 2, 2))(self.b_back2))
self.b_back1 = layers.BatchNormalization()(self.b_back1)
self.gb = layers.GlobalAveragePooling3D()(self.b_back1)
self.dr = layers.Dropout(rate=dr)(self.gb)
self.dense = layers.Dense(1, activation='sigmoid')(self.dr)
self.model = keras.Model(input=self.inputs, output=self.dense)
def D3GenerateModel(n_filter=16, number_of_class=1, input_shape=(16,144,144,1),activation_last='softmax', metrics=['mse', 'acc', dice_coef, recall_at_thresholds, precision_at_thresholds], loss='categorical_crossentropy', dropout=0.05, init='glorot_uniform', two_output=False):
#init = initializers.VarianceScaling(scale=1.0, mode='fan_in', distribution='normal', seed=None)
filter_size =n_filter
input_x = layers.Input(shape=input_shape,name='Input_layer', dtype = 'float32')
#1 level
x = layers.Conv3D(filters=filter_size, kernel_size=(5,5,5), strides = (1,1,1), kernel_initializer=init, padding='same')(input_x)
x = cyclical_learning_rate.SineReLU()(x)
x = layers.Conv3D(filters=filter_size, kernel_size=(5,5,5), strides=(1,1, 1),
padding='same',kernel_initializer=init)(x)
x = cyclical_learning_rate.SineReLU()(x)
x = layers.MaxPooling3D(pool_size=(2,2,2), padding='same')(x)
#2 level
conv_list = []
counter = 0
x = layers.Conv3D(filters=filter_size*2, kernel_size=(3,3,3), strides=(1,1, 1),
padding='same',kernel_initializer=init)(x)
x = cyclical_learning_rate.SineReLU()(x)
x = layers.Conv3D(filters=filter_size*2, kernel_size=(3,3,3), strides=(1,1, 1),
padding='same',kernel_initializer=init)(x)
x = cyclical_learning_rate.SineReLU()(x)
x = layers.AveragePooling3D(pool_size=(1,2,2), padding='same')(x)
x = layers.UpSampling3D(size=(1,2,2))(x)
for index ,kernel_sizes in enumerate([
[(1,3,3), (3,3,1)], #Changed [(1,3,3), (1,1,3)]
[(3,3,3), (3,1,3)], #Changed [(3,3,3), (3,1,3)]
[(3,3,1), (3,3,3), (1,3,3)] #Changed [(3,3,1), (1,3,1)]
]):
for kernel_size in (kernel_sizes):
x = layers.Conv3D(filters=(filter_size*4), kernel_size=kernel_size, kernel_initializer=init, strides =(1,1,1), padding='same', name='Conv3D_%s' % (counter))(x)
x = layers.BatchNormalization()(x)
x = cyclical_learning_rate.SineReLU()(x)
counter = counter+1
conv_list.append(x)
x = layers.concatenate(conv_list)
x = layers.Conv3D(filters=filter_size*8, kernel_size=(3,3,3), strides=(2,2, 2), kernel_initializer=init,
padding='same')(x)
x = layers.BatchNormalization()(x)
x = cyclical_learning_rate.SineReLU()(x)
#x = layers.MaxPooling3D(pool_size=(2,2, 2))(x)
x = layers.Reshape(target_shape=[4,-1, filter_size*8])(x)
x = layers.Conv2D(filters=filter_size*8, kernel_size=(1,1296), kernel_initializer=init, strides=(1,1296))(x)
x = layers.BatchNormalization()(x)
x = cyclical_learning_rate.SineReLU()(x)
x = layers.Reshape(target_shape=[filter_size*8,-1])(x)
x = layers.Conv1D(filters=2, kernel_size=filter_size*8, strides=filter_size*8, kernel_initializer=init)(x)
x = layers.Softmax()(x)
y = layers.Flatten()(x)
#Classification
model = Model(inputs=input_x, outputs=y)
#optimizer = tf.contrib.opt.AdamWOptimizer(weight_decay=0.000001,lr=lr)
#keras.optimizers.SGD(lr=lr, momentum=0.90, decay=decay, nesterov=False)
#opt_noise = add_gradient_noise(optimizers.Adam)
#optimizer = 'Adam'#opt_noise(lr, amsgrad=True)#, nesterov=True)#opt_noise(lr, amsgrad=True)
import yogi
optimizer = yogi.Yogi(lr=lr)
#optimizer=optimizers.adam(lr, amsgrad=True)
model.compile(optimizer=optimizer,loss=loss, metrics=metrics)#categorical_crossentropy
return model
def create_autoencoder():
model = models.Sequential()
model.add(
layers.Conv3D(64,
5,
padding='same',
activation='relu',
input_shape=(80, 80, 64, 1)))
model.add(layers.MaxPool3D())
model.add(layers.Conv3D(128, 5, padding='same', activation='relu'))
model.add(layers.MaxPool3D())
model.add(layers.Conv3D(128, 5, padding='same', activation='relu'))
model.add(layers.MaxPool3D())
model.add(layers.Conv3D(256, 5, padding='same', activation='relu'))
model.add(layers.Deconv3D(256, 4, padding='same', activation='relu'))
model.add(layers.UpSampling3D())
model.add(layers.Deconv3D(128, 4, padding='same', activation='relu'))
model.add(layers.UpSampling3D())
model.add(layers.Deconv3D(128, 4, padding='same', activation='relu'))
model.add(layers.UpSampling3D())
model.add(layers.Deconv3D(64, 4, padding='same', activation='relu'))
model.add(layers.Deconv3D(1, 4, padding='same', activation='relu'))
return model
def build_cnn(optimizer='adam', lr=0.00002):
"""Main class for setting up a CNN. Returns the compiled model."""
importlib.reload(config)
C = config.Config()
proj = layers.Input(C.proj_dims)
#x = layers.Permute((2,1,3))(img)
x = layers.Reshape((C.proj_dims[0], -1))(proj)
x = layers.Dense(1024, activation='tanh')(
x) #, kernel_regularizer=regularizers.l1(0.01)
x = layers.BatchNormalization()(x)
#x = layers.Reshape((C.proj_dims[0],32,-1))(x)
#x = layers.Conv2D(128, 3, activation='relu', padding='same')(x)
#x = layers.Reshape((C.proj_dims[0],-1))(x)
x = layers.Dense(1024, activation='tanh')(x)
x = layers.BatchNormalization()(x)
x = layers.Reshape((C.proj_dims[0], 32, 32, -1))(x)
x = layers.Conv3D(64, 3, activation='relu', padding='same')(x)
#x = layers.UpSampling3D((1,2,2))(x)
x = layers.MaxPooling3D((2, 1, 1))(x)
x = layers.Conv3D(64, 3, activation='relu', padding='same')(x)
x = layers.BatchNormalization()(x)
x = layers.Conv3DTranspose(1, 3, activation='sigmoid', padding='same')(x)
img = layers.Reshape(C.world_dims)(x)
#x = layers.Lambda(norm)(x)
#x = layers.Permute((2,1,3))(x)
#x = layers.Conv2D(64, (2,2), activation='relu', padding='same')(x)
#x = layers.Conv2D(64, (2,2), padding='same')(x)
model = Model(proj, img)
model.compile(optimizer=RMSprop(lr=lr, decay=0.1), loss='mse')
if False:
x = layers.Reshape((C.proj_dims[0], -1))(proj)
x = layers.Dense(1024, activation='tanh')(x)
x = layers.BatchNormalization()(x)
x = layers.Dense(1024, activation='tanh')(x)
x = layers.BatchNormalization()(x)
x = layers.Reshape((C.proj_dims[0], 32, 32, -1))(x)
x = layers.Conv3D(64, (3, 3, 3), activation='relu', padding='same')(x)
x = layers.UpSampling3D((1, 2, 2))(x)
x = layers.Conv3D(64, (3, 3, 3), activation='relu', padding='same')(x)
x = layers.BatchNormalization()(x)
x = layers.Conv3DTranspose(1, (1, 3, 3),
activation='sigmoid',
padding='same')(x)
return model
def __init__(self, shape):
"""
将dropout输出的结果直接影像最终结果
:param shape:
"""
self.re_rate = 0.6
self.inputs = layers.Input(shape=shape)
self.f_block = layers.Conv3D(4, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.inputs)
self.bn = layers.BatchNormalization()(self.f_block)
self.mp1 = layers.MaxPooling3D((2, 2, 2))(self.bn)
self.f_block1 = layers.Conv3D(8, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.mp1)
self.bn = layers.BatchNormalization()(self.f_block1)
self.mp2 = layers.MaxPooling3D((2, 2, 2))(self.bn)
self.f_block2 = layers.Conv3D(16, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.mp2)
self.f_block2 = layers.BatchNormalization()(self.f_block2)
self.b_back2 = layers.Conv3D(32, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.f_block2)
self.b_back2 = layers.BatchNormalization()(self.b_back2)
self.b_back2 = layers.Conv3D(
64, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(layers.UpSampling3D((2, 2, 2))(self.f_block2))
self.b_back2 = layers.BatchNormalization()(self.b_back2)
self.cat2 = layers.concatenate([self.f_block1, self.b_back2])
self.bn = layers.BatchNormalization()(self.cat2)
self.b_back1 = layers.Conv3D(
32, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(layers.UpSampling3D((2, 2, 2))(self.bn))
self.b_back1 = layers.BatchNormalization()(self.b_back1)
self.gb = layers.GlobalAveragePooling3D()(self.b_back1)
self.gb_drop = layers.Dropout(rate=0.9)(self.gb)
self.pure_dense = layers.Dense(1, activation='sigmoid')(self.gb_drop)
# add mmse
mmse_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(mmse_input)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
emCon = layers.Flatten()(embedded_layer)
self.drop = emCon
# add sex
sex_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(sex_input)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
emCon = layers.Flatten()(embedded_layer)
self.drop = layers.concatenate([self.drop, emCon])
# add age
age_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(age_input)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
emCon = layers.Flatten()(embedded_layer)
self.drop = layers.concatenate([self.drop, emCon])
# add marriage
marriage_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(marriage_input)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
emCon = layers.Flatten()(embedded_layer)
self.drop = layers.concatenate([self.drop, emCon])
# add apoe4
apoe4_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(apoe4_input)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
emCon = layers.Flatten()(embedded_layer)
self.drop = layers.concatenate([self.drop, emCon])
# add education
edu_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(edu_input)
embedded_layer = layers.Conv1D(4, 1, activation='relu')(embedded_layer)
emCon = layers.Flatten()(embedded_layer)
self.drop = layers.concatenate([self.drop, emCon])
self.drop = layers.concatenate([self.gb_drop, self.drop])
self.dense = layers.Dense(1, activation='sigmoid')(self.drop)
self.model = keras.Model(input=[
self.inputs, mmse_input, sex_input, age_input, marriage_input,
apoe4_input, edu_input
],
output=[self.pure_dense, self.dense])
def build(self):
# Initialisation
input_layer = kl.Input(shape=self.input_shape, name="input_layer")
cur_layer = input_layer
levels = []
# Downward
for i in range(self.depth):
n_kernels = self.n_base_filters * (2**i)
conv_a = kl.Conv3D(n_kernels,
kernel_size=3,
activation="relu",
padding="same",
name=f"conv_a_{i}")(cur_layer)
conv_b = kl.Conv3D(n_kernels,
kernel_size=3,
activation="relu",
padding="same",
name=f"conv_b_{i}")(conv_a)
if i < self.depth - 1:
pool = kl.MaxPool3D(pool_size=(2, 2, 2),
padding="same",
name=f"pool_{i}")(conv_b)
cur_layer = pool
levels.append([conv_a, conv_b, pool])
else:
cur_layer = conv_b
levels.append([conv_a, conv_b])
# Upward
for i in range(self.depth - 2, -1, -1):
n_kernels = self.n_base_filters * (2**i)
up = kl.UpSampling3D(size=(2, 2, 2),
name=f"upsampling_{i}")(cur_layer)
upconv_a = kl.Conv3D(n_kernels,
kernel_size=3,
activation="relu",
padding="same",
name=f"upconv_a_{i}")(up)
merge = kl.Concatenate(axis=4,
name=f"merge{i}")([levels[i][1], upconv_a])
upconv_b = kl.Conv3D(n_kernels,
kernel_size=3,
activation="relu",
padding="same",
name=f"upconv_b_{i}")(merge)
upconv_c = kl.Conv3D(n_kernels,
kernel_size=3,
activation="relu",
padding="same",
name=f"upconv_c_{i}")(upconv_b)
cur_layer = upconv_c
# Finalisation
output_layer = kl.Conv3D(self.n_classes,
1,
activation="sigmoid",
name="output_layer")(cur_layer)
model = Model(inputs=input_layer, outputs=output_layer)
return model
def __init__(self, shape):
self.re_rate = 0.9
self.inputs = layers.Input(shape=shape)
self.f_block = layers.Conv3D(4, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.inputs)
self.bn = layers.BatchNormalization()(self.f_block)
self.mp1 = layers.MaxPooling3D((2, 2, 2))(self.bn)
self.f_block1 = layers.Conv3D(16, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.mp1)
self.bn = layers.BatchNormalization()(self.f_block1)
self.f_block1 = layers.Conv3D(16, (1, 1, 1),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.bn)
self.f_block1 = layers.BatchNormalization()(self.f_block1)
self.mp2 = layers.MaxPooling3D((2, 2, 2))(self.f_block1)
self.f_block2 = layers.Conv3D(32, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.mp2)
self.f_block2 = layers.BatchNormalization()(self.f_block2)
self.f_block2 = layers.Conv3D(32, (1, 1, 1),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.f_block2)
self.f_block2 = layers.BatchNormalization()(self.f_block2)
self.mp3 = layers.MaxPooling3D((2, 2, 2))(self.f_block2)
self.f_block3 = layers.Conv3D(64, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.mp3)
self.f_block3 = layers.BatchNormalization()(self.f_block3)
self.f_block3 = layers.Conv3D(64, (1, 1, 1),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.f_block3)
self.f_block3 = layers.BatchNormalization()(self.f_block3)
# self.mp4 = layers.MaxPooling3D((2, 2, 2))(self.f_block3)
self.b_back3 = layers.Conv3D(64, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.f_block3)
self.b_back3 = layers.BatchNormalization()(self.b_back3)
self.b_back3 = layers.Conv3D(64, (1, 1, 1),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.b_back3)
self.b_back3 = layers.BatchNormalization()(self.b_back3)
self.cat1 = layers.concatenate([self.f_block3, self.b_back3])
self.bn4 = layers.BatchNormalization()(self.cat1)
self.b_back2 = layers.Conv3D(
64, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(layers.UpSampling3D((2, 2, 2))(self.bn4))
self.b_back2 = layers.BatchNormalization()(self.b_back2)
self.b_back2 = layers.Conv3D(64, (1, 1, 1),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.b_back2)
self.b_back2 = layers.BatchNormalization()(self.b_back2)
self.cat2 = layers.concatenate([self.mp2, self.b_back2])
self.bn = layers.BatchNormalization()(self.cat2)
self.b_back1 = layers.Conv3D(
32, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(layers.UpSampling3D((2, 2, 2))(self.bn))
self.b_back1 = layers.BatchNormalization()(self.b_back1)
self.b_back1 = layers.Conv3D(32, (1, 1, 1),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.b_back1)
self.b_back1 = layers.BatchNormalization()(self.b_back1)
self.cat3 = layers.concatenate([self.mp1, self.b_back1])
self.gb = layers.GlobalAveragePooling3D()(self.cat3)
self.dense3 = layers.Dense(1, activation='sigmoid')(self.gb)
self.model = keras.Model(input=self.inputs, output=self.dense3)
layers.Dense(np.prod(input_shape) / 1728,
activation='relu',
input_dim=4 * n_class))
decoder.add(
layers.Reshape(target_shape=(52 / (RzFaktor), Ry / 36, Rx / 48, 3),
name='out_recon_unupsized1'))
decoder.add(
layers.BatchNormalization(epsilon=0.001,
axis=-1,
momentum=0.99,
weights=None,
beta_init='zero',
gamma_init='one',
gamma_regularizer=None,
beta_regularizer=None))
decoder.add(layers.UpSampling3D(size=(1, 3, 4)))
decoder.add(
layers.Conv3D(filters=32,
kernel_size=(3, 5, 5),
strides=1,
padding='same',
activation='relu',
name='conv3Dout0'))
decoder.add(
layers.BatchNormalization(epsilon=0.001,
axis=-1,
momentum=0.99,
weights=None,
beta_init='zero',
gamma_init='one',
gamma_regularizer=None,
def Conv_VAE3D(n_epochs=2,
batch_size=10,
learning_rate=0.001,
decay_rate=0.0,
latent_dim=8,
name='stats.pickle'):
# Prepare session:
K.clear_session()
# Number of samples to use for training and validation:
n_train = 1500
n_val = 1000
# ENCODER: ---------------------------------------------------------------
input_img = Input(shape=(50, 50, 50, 4), name="Init_Input")
x = layers.Conv3D(32, (3, 3, 3),
padding="same",
activation='relu',
name='E_Conv1')(input_img)
x = layers.MaxPooling3D((2, 2, 2), name='E_MP1')(x)
x = layers.Conv3D(64, (3, 3, 3),
padding="same",
activation='relu',
name='E_Conv2')(x)
x = layers.MaxPooling3D((2, 2, 2), name='E_MP2')(x)
x = layers.Conv3D(64, (3, 3, 3),
padding="valid",
activation='relu',
name='E_Conv3')(x)
x = layers.MaxPooling3D((2, 2, 2), name='E_MP3')(x)
x = layers.Conv3D(128, (3, 3, 3),
padding="same",
activation='relu',
name='E_Conv4')(x)
shape_before_flattening = K.int_shape(x)
x = layers.Flatten()(x)
x = layers.Dense(32, activation='relu')(x)
encoder = Model(input_img, x)
print(encoder.summary())
# VARIATIONAL LAYER: ------------------------------------------------------
z_mean = layers.Dense(latent_dim, name='V_Mean')(x)
z_log_var = layers.Dense(latent_dim, name='V_Sig')(x)
def sampling(args):
z_mean, z_log_var = args
epsilon = K.random_normal(shape=(K.shape(z_mean)[0], latent_dim),
mean=0.,
stddev=1.)
return z_mean + K.exp(z_log_var) * epsilon
z = layers.Lambda(sampling, name='V_Var')([z_mean, z_log_var])
variation = Model(input_img, z)
print(variation.summary())
# DECODER: ---------------------------------------------------------------
decoder_input = layers.Input(shape=(latent_dim, ), name='D_Input')
x = layers.Dense(np.prod(shape_before_flattening[1:]),
activation='relu',
name='D_Dense')(decoder_input)
x = layers.Reshape(shape_before_flattening[1:], name='D_UnFlatten')(x)
x = layers.Conv3DTranspose(32,
3,
padding='same',
activation='relu',
name='D_DeConv1')(x)
x = layers.UpSampling3D((2, 2, 2))(x)
x = layers.Conv3D(4,
3,
padding='same',
activation='sigmoid',
name='D_Conv1')(x)
x = layers.UpSampling3D((5, 5, 5))(x)
x = layers.Conv3D(4,
3,
padding='same',
activation='sigmoid',
name='D_Conv2')(x)
decoder = Model(decoder_input, x)
print(decoder.summary())
# CALLBACKS: --------------------------------------------------------------
class TimeHistory(keras.callbacks.Callback):
start = []
end = []
times = []
def on_epoch_begin(self, batch, logs=None):
self.start = time.time()
def on_epoch_end(self, batch, logs=None):
self.end = time.time()
self.times.append(self.end - self.start)
# CUSTOM LAYERS: ----------------------------------------------------------
class CustomVariationalLayer(keras.layers.Layer):
def vae_loss(self, x, z_decoded):
x = K.flatten(x)
z_decoded = K.flatten(z_decoded)
xent_loss = keras.metrics.binary_crossentropy(x, z_decoded)
kl_loss = -5e-4 * K.mean(
1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return K.mean(xent_loss + kl_loss) #xent_loss) # + kl_loss)
def call(self, inputs):
x = inputs[0]
z_decoded = inputs[1]
loss = self.vae_loss(x, z_decoded)
self.add_loss(loss, inputs=inputs)
return x
# DEFINE FINAL MODEL: ----------------------------------------------------
z_encoded = variation(input_img)
z_decoded = decoder(z_encoded)
# Construct Final Model:
y = CustomVariationalLayer()([input_img, z_decoded])
vae = Model(input_img, y)
print(vae.summary())
# Define Optimizer:
vae_optimizer = keras.optimizers.Adam(lr=learning_rate,
beta_1=0.9,
beta_2=0.999,
decay=decay_rate,
amsgrad=False)
vae.compile(optimizer=vae_optimizer,
loss=None) # Not using custom vae loss function defined above
# Define time callback:
time_callback = TimeHistory()
steps = n_train // batch_size
val_steps = n_val // batch_size
# FIT MODEL: --------------------------------------------------------------
history = vae.fit_generator(
gen_batches(batch_size),
shuffle=True,
epochs=n_epochs,
steps_per_epoch=steps,
callbacks=[time_callback],
validation_data=gen_batches_validation(batch_size),
validation_steps=val_steps)
# OUTPUTS: -------------------------------------------------------------
history_dict = history.history
loss_values = history_dict['loss']
val_loss_values = history_dict['val_loss']
times = time_callback.times
data = {
'train_loss': loss_values,
'val_loss': val_loss_values,
'epoch_time': times
}
pickle_out = open(name, "wb")
pickle.dump(data, pickle_out)
pickle_out.close()
K.clear_session()
return (history_dict)
def __init__(self, shape):
"""
更改临床数据的传入位置
在主管上，我们给出评级：sex,apoe4对于脑部区域有着明显的影响，并且与MRI影响有着强相关关系，应该在临床数据使用的前面
age,marriage,edu：理论上影响不是很大，是社会对于精神的影响可以放在第二层次
mmse：对最终结果有着明显的指向
:param shape:
"""
self.re_rate = 0.6
self.inputs = layers.Input(shape=shape)
self.f_block = layers.Conv3D(4, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.inputs)
self.bn = layers.BatchNormalization()(self.f_block)
self.mp1 = layers.MaxPooling3D((2, 2, 2))(self.bn)
self.f_block1 = layers.Conv3D(8, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.mp1)
self.bn = layers.BatchNormalization()(self.f_block1)
self.mp2 = layers.MaxPooling3D((2, 2, 2))(self.bn)
self.f_block2 = layers.Conv3D(16, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.mp2)
self.f_block2 = layers.BatchNormalization()(self.f_block2)
self.b_back2 = layers.Conv3D(32, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(
self.re_rate),
padding='same')(self.f_block2)
self.b_back2 = layers.BatchNormalization()(self.b_back2)
self.b_back2 = layers.Conv3D(
32, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(layers.UpSampling3D((2, 2, 2))(self.f_block2))
self.b_back2 = layers.BatchNormalization()(self.b_back2)
self.cat2 = layers.concatenate([self.f_block1, self.b_back2])
self.bn = layers.BatchNormalization()(self.cat2)
self.b_back1 = layers.Conv3D(
32, (3, 3, 3),
activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(layers.UpSampling3D((2, 2, 2))(self.bn))
self.b_back1 = layers.BatchNormalization()(self.b_back1)
self.gb = layers.GlobalAveragePooling3D()(self.b_back1)
self.drop = layers.Dropout(rate=0.9)(self.gb)
# add apoe4
apoe4_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(apoe4_input)
emCon = layers.Flatten()(embedded_layer)
self.drop = layers.concatenate([self.drop, emCon])
# add sex
sex_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(sex_input)
emCon = layers.Flatten()(embedded_layer)
self.dense = layers.concatenate([self.drop, emCon])
self.dense = layers.Dense(32, activation='relu')(self.dense)
# add age
age_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(age_input)
emCon = layers.Flatten()(embedded_layer)
self.dense = layers.concatenate([self.dense, emCon])
# add marriage
marriage_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(marriage_input)
emCon = layers.Flatten()(embedded_layer)
self.dense = layers.concatenate([self.dense, emCon])
# add education
edu_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(edu_input)
emCon = layers.Flatten()(embedded_layer)
self.dense = layers.concatenate([self.dense, emCon])
self.dense = layers.Dense(16, activation='relu')(self.dense)
# add mmse
mmse_input = layers.Input(shape=(1, ), dtype='int32')
embedded_layer = layers.Embedding(shape[-1], 1)(mmse_input)
emCon = layers.Flatten()(embedded_layer)
self.dense = layers.concatenate([self.dense, emCon])
self.dense = layers.Dense(1, activation='sigmoid')(self.dense)
self.model = keras.Model(input=[
self.inputs, mmse_input, sex_input, age_input, marriage_input,
apoe4_input, edu_input
],
output=self.dense)
def build(self, mode, config):
"""Build Mask R-CNN architecture.
input_shape: The shape of the input image.
mode: Either "training" or "inference". The inputs and
outputs of the model differ accordingly.
"""
assert mode in ['training', 'inference']
# Image size must be dividable by 2 multiple times
h, w, z = config.IMAGE_SHAPE[:3]
if h / 2**6 != int(h / 2**6) or w / 2**6 != int(
w / 2**6) or z / 2**5 != int(z / 2**5):
raise Exception(
"Image size must be dividable by 2 at least 6 times "
"to avoid fractions when downscaling and upscaling."
"For example, use 256, 320, 384, 448, 512, ... etc. ")
# Inputs
input_image = KL.Input(shape=[None, None, None, 3], name="input_image")
input_image_meta = KL.Input(shape=[config.IMAGE_META_SIZE],
name="input_image_meta")
if mode == "training":
# RPN GT
input_rpn_match = KL.Input(shape=[None, 1],
name="input_rpn_match",
dtype=tf.int32)
input_rpn_bbox = KL.Input(shape=[None, 6],
name="input_rpn_bbox",
dtype=tf.float32)
# Detection GT (class IDs, bounding boxes, and masks)
# 1. GT Class IDs (zero padded)
input_gt_class_ids = KL.Input(shape=[None],
name="input_gt_class_ids",
dtype=tf.int32)
# 2. GT Boxes in pixels (zero padded)
# MODIFIED FOR 3D
# [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)] in image coordinates
input_gt_boxes = KL.Input(shape=[None, 6],
name="input_gt_boxes",
dtype=tf.float32)
# Normalize coordinates
gt_boxes = KL.Lambda(lambda x: norm_boxes_graph(
x,
K.shape(input_image)[1:4]))(input_gt_boxes)
# 3. GT Masks (zero padded)
# [batch, height, width, MAX_GT_INSTANCES]
# MODIFIED FOR 3D
#if config.USE_MINI_MASK:
# input_gt_masks = KL.Input(
# shape=[config.MINI_MASK_SHAPE[0],
# config.MINI_MASK_SHAPE[1], None],
# name="input_gt_masks", dtype=bool)
#else:
input_gt_masks = KL.Input(shape=[
config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1],
config.IMAGE_SHAPE[2], None
],
name="input_gt_masks",
dtype=bool)
elif mode == "inference":
# Anchors in normalized coordinates
input_anchors = KL.Input(shape=[None, 4], name="input_anchors")
# Build the shared convolutional layers.
# Bottom-up Layers
# Returns a list of the last layers of each stage, 5 in total.
# Don't create the thead (stage 5), so we pick the 4th item in the list.
if callable(config.BACKBONE):
_, C2, C3, C4, C5 = config.BACKBONE(input_image,
stage5=True,
train_bn=config.TRAIN_BN)
else:
_, C2, C3, C4, C5 = resnet_graph(input_image,
config.BACKBONE,
stage5=True,
train_bn=config.TRAIN_BN)
# Top-down Layers
# TODO: add assert to varify feature map sizes match what's in config
P5 = KL.Conv3D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1, 1),
name='fpn_c5p5')(C5)
P4 = KL.Add(name="fpn_p4add")([
KL.UpSampling3D(size=(2, 2, 2), name="fpn_p5upsampled")(P5),
KL.Conv3D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1, 1),
name='fpn_c4p4')(C4)
])
P3 = KL.Add(name="fpn_p3add")([
KL.UpSampling3D(size=(2, 2, 2), name="fpn_p4upsampled")(P4),
KL.Conv3D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1, 1),
name='fpn_c3p3')(C3)
])
P2 = KL.Add(name="fpn_p2add")([
KL.UpSampling3D(size=(2, 2, 2), name="fpn_p3upsampled")(P3),
KL.Conv3D(config.TOP_DOWN_PYRAMID_SIZE, (1, 1, 1),
name='fpn_c2p2')(C2)
])
# Attach 3x3 conv to all P layers to get the final feature maps.
P2 = KL.Conv3D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3, 3),
padding="SAME",
name="fpn_p2")(P2)
P3 = KL.Conv3D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3, 3),
padding="SAME",
name="fpn_p3")(P3)
P4 = KL.Conv3D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3, 3),
padding="SAME",
name="fpn_p4")(P4)
P5 = KL.Conv3D(config.TOP_DOWN_PYRAMID_SIZE, (3, 3, 3),
padding="SAME",
name="fpn_p5")(P5)
# P6 is used for the 5th anchor scale in RPN. Generated by
# subsampling from P5 with stride of 2.
P6 = KL.MaxPooling3D(pool_size=(1, 1, 1), strides=2, name="fpn_p6")(P5)
# Note that P6 is used in RPN, but not in the classifier heads.
rpn_feature_maps = [P2, P3, P4, P5, P6]
mrcnn_feature_maps = [P2, P3, P4, P5]
# Anchors
if mode == "training":
anchors = self.get_anchors(config.IMAGE_SHAPE)
# Duplicate across the batch dimension because Keras requires it
# TODO: can this be optimized to avoid duplicating the anchors?
anchors = np.broadcast_to(anchors,
(config.BATCH_SIZE, ) + anchors.shape)
# A hack to get around Keras's bad support for constants
anchors = KL.Lambda(lambda x: tf.Variable(anchors),
name="anchors")(input_image)
else:
anchors = input_anchors
# RPN Model
rpn = build_rpn_model(config.RPN_ANCHOR_STRIDE,
len(config.RPN_ANCHOR_RATIOS),
config.TOP_DOWN_PYRAMID_SIZE)
# Loop through pyramid layers
layer_outputs = [] # list of lists
for p in rpn_feature_maps:
layer_outputs.append(rpn([p]))
# Concatenate layer outputs
# Convert from list of lists of level outputs to list of lists
# of outputs across levels.
# e.g. [[a1, b1, c1], [a2, b2, c2]] => [[a1, a2], [b1, b2], [c1, c2]]
output_names = ["rpn_class_logits", "rpn_class", "rpn_bbox"]
outputs = list(zip(*layer_outputs))
outputs = [
KL.Concatenate(axis=1, name=n)(list(o))
for o, n in zip(outputs, output_names)
]
rpn_class_logits, rpn_class, rpn_bbox = outputs
# Generate proposals
# Proposals are [batch, N, (y1, x1, y2, x2)] in normalized coordinates
# and zero padded.
proposal_count = config.POST_NMS_ROIS_TRAINING if mode == "training"\
else config.POST_NMS_ROIS_INFERENCE
rpn_rois = ProposalLayer(proposal_count=proposal_count,
nms_threshold=config.RPN_NMS_THRESHOLD,
name="ROI",
config=config)([rpn_class, rpn_bbox, anchors])
"""
if mode == "training":
# Class ID mask to mark class IDs supported by the dataset the image
# came from.
active_class_ids = KL.Lambda(
lambda x: parse_image_meta_graph(x)["active_class_ids"]
)(input_image_meta)
if not config.USE_RPN_ROIS:
# Ignore predicted ROIs and use ROIs provided as an input.
input_rois = KL.Input(shape=[config.POST_NMS_ROIS_TRAINING, 4],
name="input_roi", dtype=np.int32)
# Normalize coordinates
target_rois = KL.Lambda(lambda x: norm_boxes_graph(
x, K.shape(input_image)[1:3]))(input_rois)
else:
target_rois = rpn_rois
# Generate detection targets
# Subsamples proposals and generates target outputs for training
# Note that proposal class IDs, gt_boxes, and gt_masks are zero
# padded. Equally, returned rois and targets are zero padded.
rois, target_class_ids, target_bbox, target_mask =\
DetectionTargetLayer(config, name="proposal_targets")([
target_rois, input_gt_class_ids, gt_boxes, input_gt_masks])
# Network Heads
# TODO: verify that this handles zero padded ROIs
mrcnn_class_logits, mrcnn_class, mrcnn_bbox =\
fpn_classifier_graph(rois, mrcnn_feature_maps, input_image_meta,
config.POOL_SIZE, config.NUM_CLASSES,
train_bn=config.TRAIN_BN,
fc_layers_size=config.FPN_CLASSIF_FC_LAYERS_SIZE)
mrcnn_mask = build_fpn_mask_graph(rois, mrcnn_feature_maps,
input_image_meta,
config.MASK_POOL_SIZE,
config.NUM_CLASSES,
train_bn=config.TRAIN_BN)
# TODO: clean up (use tf.identify if necessary)
output_rois = KL.Lambda(lambda x: x * 1, name="output_rois")(rois)
# Losses
rpn_class_loss = KL.Lambda(lambda x: rpn_class_loss_graph(*x), name="rpn_class_loss")(
[input_rpn_match, rpn_class_logits])
rpn_bbox_loss = KL.Lambda(lambda x: rpn_bbox_loss_graph(config, *x), name="rpn_bbox_loss")(
[input_rpn_bbox, input_rpn_match, rpn_bbox])
class_loss = KL.Lambda(lambda x: mrcnn_class_loss_graph(*x), name="mrcnn_class_loss")(
[target_class_ids, mrcnn_class_logits, active_class_ids])
bbox_loss = KL.Lambda(lambda x: mrcnn_bbox_loss_graph(*x), name="mrcnn_bbox_loss")(
[target_bbox, target_class_ids, mrcnn_bbox])
mask_loss = KL.Lambda(lambda x: mrcnn_mask_loss_graph(*x), name="mrcnn_mask_loss")(
[target_mask, target_class_ids, mrcnn_mask])
# Model
inputs = [input_image, input_image_meta,
input_rpn_match, input_rpn_bbox, input_gt_class_ids, input_gt_boxes, input_gt_masks]
if not config.USE_RPN_ROIS:
inputs.append(input_rois)
outputs = [rpn_class_logits, rpn_class, rpn_bbox,
mrcnn_class_logits, mrcnn_class, mrcnn_bbox, mrcnn_mask,
rpn_rois, output_rois,
rpn_class_loss, rpn_bbox_loss, class_loss, bbox_loss, mask_loss]
model = KM.Model(inputs, outputs, name='mask_rcnn')
""" """
def buildModelFCN3D(inpShape=(256, 256, 128, 1),
numCls=2,
numConv=1,
kernelSize=3,
numFlt=16,
ppad='same',
numSubsampling=5,
isUnet=True,
isDebug=False,
isSkipFirstConnection=False):
dataInput = kl.Input(shape=inpShape)
fsiz = (kernelSize, kernelSize, kernelSize)
psiz = (2, 2, 2)
x = dataInput
# -------- Encoder --------
lstMaxPools = []
for cc in range(numSubsampling):
for ii in range(numConv):
x = kl.Conv3D(filters=numFlt * (2**cc),
kernel_size=fsiz,
padding=ppad,
activation='relu')(x)
lstMaxPools.append(x)
x = kl.MaxPooling3D(pool_size=psiz)(x)
# -------- Decoder --------
for cc in range(numSubsampling):
for ii in range(numConv):
x = kl.Conv3D(filters=numFlt * (2**(numSubsampling - 1 - cc)),
kernel_size=fsiz,
padding=ppad,
activation='relu')(x)
x = kl.UpSampling3D(size=psiz)(x)
if isUnet:
if isSkipFirstConnection and (cc == (numSubsampling - 1)):
continue
x = kl.concatenate([x, lstMaxPools[-1 - cc]], axis=-1)
# final convs
for cc in range(numSubsampling):
x = kl.Conv3D(filters=numFlt,
kernel_size=fsiz,
padding=ppad,
activation='relu')(x)
# 1x1 Convolution: emulation of Dense layer
if numCls == 2:
x = kl.Conv3D(filters=1,
kernel_size=(1, 1, 1),
padding='valid',
activation='sigmoid',
name='conv3d_out_c{}'.format(numCls))(x)
x = kl.Reshape([inpShape[0], inpShape[1], inpShape[2]])(x)
else:
x = kl.Conv3D(filters=numCls,
kernel_size=(1, 1, 1),
padding='valid',
activation='linear',
name='conv3d_out_c{}'.format(numCls))(x)
x = kl.Reshape([-1, numCls])(x)
x = kl.Activation('softmax')(x)
x = kl.Reshape([inpShape[0], inpShape[1], inpShape[2], numCls])(x)
retModel = keras.models.Model(dataInput, x)
if isDebug:
retModel.summary()
fimg_model = 'model_graph_FCNN3D.png'
keras.utils.plot_model(retModel, fimg_model, show_shapes=True)
plt.imshow(plt.imread(fimg_model))
plt.show()
return retModel
def fit(self, training_set, training_data_name=None):
import tensorflow as tf
from keras.backend.tensorflow_backend import set_session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
set_session(sess)
temp_training_set = []
if training_data_name is not None:
self.training_data_name = training_data_name
if any(isinstance(el, list) for el in
training_set): # if training set is a sequence of frames
for sequence in training_set:
transformed_frames = self.input_frames_transform(
np.array(sequence))
temp_training_set += self.get_training_set(transformed_frames)
else:
transformed_frames = self.input_frames_transform(
np.array(training_set))
temp_training_set = self.get_training_set(transformed_frames)
final_training_set = np.array(temp_training_set)
seq = Sequential()
seq.add(
layers.Conv3D(16, (3, 3, 3),
activation='relu',
padding='same',
batch_input_shape=(None, 12, 256, 256, 1)))
seq.add(layers.Conv3D(16, (3, 3, 3), activation='relu',
padding='same'))
seq.add(layers.MaxPooling3D((2, 2, 2), padding='same'))
seq.add(layers.Conv3D(8, (3, 3, 3), activation='relu', padding='same'))
seq.add(layers.MaxPooling3D((2, 2, 2), padding='same'))
seq.add(layers.Conv3D(8, (3, 3, 3), activation='relu', padding='same'))
seq.add(layers.UpSampling3D((2, 2, 2)))
seq.add(layers.Conv3D(16, (3, 3, 3), activation='relu',
padding='same'))
seq.add(layers.UpSampling3D((2, 2, 2)))
seq.add(
layers.Conv3D(1, (3, 3, 3), activation='sigmoid', padding='same'))
print(seq.summary())
seq.compile(loss='mse',
optimizer=keras.optimizers.Adam(lr=1e-4,
decay=1e-5,
epsilon=1e-6))
seq.fit(final_training_set,
final_training_set,
batch_size=self.batch_size,
epochs=self.epochs,
shuffle=False)
self.model = seq
self.save_model()
def build_model(cfg,
num_conv=2,
kernelSize=3,
num_flt=16,
ppad='same',
num_subs=5,
is_unet=True,
is_debug=False,
isSkipFirstConnection=False,
inp_shape=None):
"""
:type cfg: Config
"""
if inp_shape is None:
inp_shape = cfg.inp_shape
dataInput = kl.Input(shape=inp_shape)
fsiz = (kernelSize, kernelSize, kernelSize)
psiz = (2, 2, 2)
x = dataInput
# -------- Encoder --------
lstMaxPools = []
for cc in range(num_subs):
for ii in range(num_conv):
x = kl.Conv3D(filters=num_flt * (2**cc),
kernel_size=fsiz,
padding=ppad,
activation='relu')(x)
lstMaxPools.append(x)
x = kl.MaxPooling3D(pool_size=psiz)(x)
# -------- Decoder --------
for cc in range(num_subs):
for ii in range(num_conv):
x = kl.Conv3D(filters=num_flt * (2**(num_subs - 1 - cc)),
kernel_size=fsiz,
padding=ppad,
activation='relu')(x)
x = kl.UpSampling3D(size=psiz)(x)
if is_unet:
if isSkipFirstConnection and (cc == (num_subs - 1)):
continue
x = kl.concatenate([x, lstMaxPools[-1 - cc]], axis=-1)
# final convs
for cc in range(num_subs):
x = kl.Conv3D(filters=num_flt,
kernel_size=fsiz,
padding=ppad,
activation='relu')(x)
# 1x1 Convolution: emulation of Dense layer
if cfg.num_cls == 2:
x = kl.Conv3D(filters=1,
kernel_size=(1, 1, 1),
padding='valid',
activation='sigmoid')(x)
x = kl.Reshape([cfg.inp_shape[0], cfg.inp_shape[1],
cfg.inp_shape[2]])(x)
else:
x = kl.Conv3D(filters=cfg.num_cls,
kernel_size=(1, 1, 1),
padding='valid',
activation='linear')(x)
x = kl.Reshape([-1, cfg.num_cls])(x)
x = kl.Activation('softmax')(x)
x = kl.Reshape([inp_shape[0], inp_shape[1], inp_shape[2],
cfg.num_cls])(x)
retModel = keras.models.Model(dataInput, x)
if is_debug:
retModel.summary()
fimg_model = 'model_graph_fcnn3d.png'
keras.utils.plot_model(retModel, fimg_model, show_shapes=True)
plt.imshow(plt.imread(fimg_model))
plt.show()
return retModel
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='c_net')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class,))
masked_by_y = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. For training
masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction
print(np.prod(input_shape))
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
#decoder.add(layers.Dense(128, activation='relu', input_dim=16*n_class))
decoder.add(layers.Dense(np.prod(input_shape)/256, activation='relu', input_dim=16*n_class))
decoder.add(layers.Reshape(target_shape=(16/(RzFaktor*4),112/8,112/8,3), name='out_recon_unupsized1'))
decoder.add(layers.BatchNormalization(epsilon=0.001, axis=-1, momentum=0.99, weights=None, beta_init='zero', gamma_init='one', gamma_regularizer=None, beta_regularizer=None))
decoder.add(layers.UpSampling3D(size=(2, 2, 2)))
decoder.add(layers.Conv3D(filters=128, kernel_size=(3,3,3), strides=1, padding='same', activation='relu', name='conv3Dout0'))
decoder.add(layers.BatchNormalization(epsilon=0.001, axis=-1, momentum=0.99, weights=None, beta_init='zero', gamma_init='one', gamma_regularizer=None, beta_regularizer=None))
decoder.add(layers.UpSampling3D(size=(2, 2, 2)))
decoder.add(layers.Conv3D(filters=64, kernel_size=(3,3,3), strides=1, padding='same', activation='relu', name='conv3Dout1'))
decoder.add(layers.BatchNormalization(epsilon=0.001, axis=-1, momentum=0.99, weights=None, beta_init='zero', gamma_init='one', gamma_regularizer=None, beta_regularizer=None))
decoder.add(layers.UpSampling3D(size=(1, 2, 2)))
decoder.add(layers.Conv3D(filters=3, kernel_size=(3,3,3), strides=1, padding='same', activation='sigmoid', name='conv3Dout2'))
# Models for training and evaluation (prediction)
model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, 16))
def build(self, mode, config):
""" Build PADNet architecture.
mode: either "training" or "inference". The inputs and outputs of the model
differ accordingly.
"""
assert mode in ['training', 'inference']
# Image size must be dividable by 2 multiple times
h, w = config.IMAGE_SHAPE[:2]
if h / 2**6 != int(h / 2**6) or w / 2**6 != int(w / 2**6):
raise Exception(
"Image size must be dividable by 2 at least 6 times "
"to avoid fractions when downscaling and upscaling."
"For example, use 256, 320, 384, 448, 512, ... etc. ")
# Inputs:
input_rgb_clip = KL.Input(shape=config.RGB_CLIP_SHAPE.tolist(),
name="input_rgb_clip")
input_flow_clip = KL.Input(shape=config.FLOW_CLIP_SHAPE.tolist(),
name="input_flow_clip")
input_image_meta = KL.Input(shape=[None], name="input_image_meta")
input_labeled_frame_id = KL.Input(shape=[config.TIMESTEPS],
name="input_labeled_frame_id",
dtype=tf.int32) # 1 frame labeled
if mode == "training":
# RPN GT
input_rpn_match = KL.Input(shape=[None, 1],
name="input_rpn_match",
dtype=tf.int32)
input_rpn_bbox = KL.Input(shape=[None, 4],
name="input_rpn_bbox",
dtype=tf.float32)
# GT Boxes (zero padded)
# [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2, actor_class_id, action_class_id)] in image coordinates
input_gt_boxes = KL.Input(shape=[None, 6],
name="input_gt_boxes",
dtype=tf.int32)
# Normalize coordinates
h, w = K.shape(input_rgb_clip)[2], K.shape(input_rgb_clip)[3]
image_scale = K.cast(K.stack([h, w, h, w, 1, 1], axis=0),
tf.float32)
gt_boxes = KL.Lambda(
lambda x: K.cast(x, tf.float32) / image_scale)(input_gt_boxes)
# GT Masks (zero padded)
# [batch, height, width, MAX_GT_INSTANCES]
if config.USE_MINI_MASK:
input_gt_masks = KL.Input(shape=[
config.MINI_MASK_SHAPE[0], config.MINI_MASK_SHAPE[1], None
],
name="input_gt_masks",
dtype=bool)
else:
input_gt_masks = KL.Input(
shape=[config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1], None],
name="input_gt_masks",
dtype=bool)
# TODO: consider using name scope for better tensorboard visualization
ri3d_end_points = i3d_graph(input_rgb_clip, mode, config,
'rgb') # TODO: mode should be inference
rC2, rC3, rC4, rC5 = ri3d_end_points['rgb_Conv3d_2c_3x3'], ri3d_end_points['rgb_Mixed_3c'], \
ri3d_end_points['rgb_Mixed_4f'], ri3d_end_points['rgb_Mixed_5c']
#shapes:
# rC2: (bs, T/2, H/4, W/4, 192)
# rC3: (bs, T/2, H/8, W/8, 480)
# rC4: (bs, T/4, H/16, W/16, 832)
# rC5: (bs, T/8, H/32, W/32, 1024)
_rP5 = KL.Conv3D(128, (1, 1, 1),
name='fpn_rc5p5')(rC5) # TODO: mask rcnn used 256
_rP4 = KL.Add(name='fpn_rp4add')([
KL.UpSampling3D(size=(1, 2, 2), name='fpn_rp5upsampled')(_rP5),
KL.Conv3D(128, (1, 1, 1), name='fpn_rc4p4')(rC4)
]) # TODO: shared conv2d layers may be better than these conv3d
_rP3 = KL.Add(name='fpn_rp3add')([
KL.UpSampling3D(size=(1, 2, 2), name='fpn_rp4upsampled')(_rP4),
KL.Conv3D(128, (1, 1, 1), name='fpn_rc3p3')(rC3)
])
_rP2 = KL.Add(name='fpn_rp2add')([
KL.UpSampling3D(size=(1, 2, 2), name='fpn_rp3upsampled')(_rP3),
KL.Conv3D(128, (1, 1, 1), name='fpn_rc2p2')(rC2)
])
# Attach 3x3x3 conv to all P layers to get the final feature maps.
rP2 = KL.Conv3D(128, (3, 3, 3), padding='SAME', name='fpn_rp2')(_rP2)
rP3 = KL.Conv3D(128, (3, 3, 3), padding='SAME', name='fpn_rp3')(_rP3)
rP4 = KL.Conv3D(128, (3, 3, 3), padding='SAME', name='fpn_rp4')(_rP4)
rP5 = KL.Conv3D(128, (3, 3, 3), padding='SAME', name='fpn_rp5')(_rP5)
fi3d_end_points = i3d_graph(input_flow_clip, mode, config,
'flow') # TODO: mode should be inference
fC2, fC3, fC4, fC5 = fi3d_end_points['flow_Conv3d_2c_3x3'], fi3d_end_points['flow_Mixed_3c'], \
fi3d_end_points['flow_Mixed_4f'], fi3d_end_points['flow_Mixed_5c']
_fP5 = KL.Conv3D(128, (1, 1, 1), name='fpn_fc5p5')(fC5)
_fP4 = KL.Add(name='fpn_fp4add')([
KL.UpSampling3D(size=(1, 2, 2), name='fpn_fp5upsampled')(_fP5),
KL.Conv3D(128, (1, 1, 1), name='fpn_fc4p4')(fC4)
]) # TODO: shared conv2d layers may be better than these conv3d
_fP3 = KL.Add(name='fpn_fp3add')([
KL.UpSampling3D(size=(1, 2, 2), name='fpn_fp4upsampled')(_fP4),
KL.Conv3D(128, (1, 1, 1), name='fpn_fc3p3')(fC3)
])
_fP2 = KL.Add(name='fpn_fp2add')([
KL.UpSampling3D(size=(1, 2, 2), name='fpn_fp3upsampled')(_fP3),
KL.Conv3D(128, (1, 1, 1), name='fpn_fc2p2')(fC2)
])
# Attach 3x3x3 conv to all P layers to get the final feature maps.
fP2 = KL.Conv3D(128, (3, 3, 3), padding='SAME', name='fpn_fp2')(_fP2)
fP3 = KL.Conv3D(128, (3, 3, 3), padding='SAME', name='fpn_fp3')(_fP3)
fP4 = KL.Conv3D(128, (3, 3, 3), padding='SAME', name='fpn_fp4')(_fP4)
fP5 = KL.Conv3D(128, (3, 3, 3), padding='SAME', name='fpn_fp5')(_fP5)
P5d, P = OrderedDict(), OrderedDict()
P5d[2] = KL.Concatenate(name='fpn_p2')([rP2, fP2])
P5d[3] = KL.Concatenate(name='fpn_p3')([rP3, fP3])
P5d[4] = KL.Concatenate(name='fpn_p4')([rP4, fP4])
P5d[5] = KL.Concatenate(name='fpn_p5')([rP5, fP5])
P5d[6] = KL.MaxPooling3D(pool_size=(1, 1, 1),
strides=(1, 2, 2),
name='fpn_p6')(P5d[5])
# TODO: better way for slicing?
def expand_inds(inds,
h=None,
w=None,
f=None,
dtype=K.dtype(input_rgb_clip)):
return K.cast(
K.tile(K.expand_dims(K.expand_dims(K.expand_dims(inds))),
[1, 1, h, w, f]), dtype)
_scale = {2: 4, 3: 8, 4: 16, 5: 32, 6: 64}
for ii in range(2, 7):
mask = KL.Lambda(expand_inds,
arguments={
'h': h // _scale[ii],
'w': w // _scale[ii],
'f': 256
})(input_labeled_frame_id)
product = KL.Multiply()([P5d[ii], mask])
P[ii] = KL.Lambda(lambda x: K.sum(x, axis=1, keepdims=False))(
product)
#KL.Multiply([P5d[ii], mask]))
if config.AGGREGATE == 'mean':
for ii in range(2, 7):
P[ii] = KL.Add()([
KL.Lambda(lambda x: K.mean(x, axis=1, keepdims=False))(
P5d[ii]), P[ii]
])
rpn_feature_maps = [P[2], P[3], P[4], P[5], P[6]]
mrcnn_feature_maps = [P[2], P[3], P[4], P[5]]
# Generate Anchors
self.anchors = matterport_utils.generate_pyramid_anchors(
config.RPN_ANCHOR_SCALES, config.RPN_ANCHOR_RATIOS,
config.BACKBONE_SHAPES, config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# RPN Model
rpn = build_rpn_model(config.RPN_ANCHOR_STRIDE,
len(config.RPN_ANCHOR_RATIOS),
256) # TODO: 256 in mask rcnn
# Loop through pyramid layers
layer_outputs = [] # list of lists
for p in rpn_feature_maps:
layer_outputs.append(rpn([p]))
# Concatenate layer outputs
# Convert from list of lists of level outputs to list of lists
# of outputs across levels.
# e.g. [[a1, b1, c1], [a2, b2, c2]] => [[a1, a2], [b1, b2], [c1, c2]]
output_names = ["rpn_class_logits", "rpn_class", "rpn_bbox"]
outputs = list(zip(*layer_outputs))
outputs = [
KL.Concatenate(axis=1, name=n)(list(o))
for o, n in zip(outputs, output_names)
]
rpn_class_logits, rpn_class, rpn_bbox = outputs
# Generate proposals
# Proposals are [N, (y1, x1, y2, x2)] in normalized coordinates.
proposal_count = config.POST_NMS_ROIS_TRAINING if mode == "training"\
else config.POST_NMS_ROIS_INFERENCE
rpn_rois = ProposalLayer(proposal_count=proposal_count,
nms_threshold=0.7,
name="ROI",
anchors=self.anchors,
config=config)([rpn_class, rpn_bbox])
if mode == "training":
# Class ID mask to mark class IDs supported by the dataset the image
# came from.
_, _, _, active_actor_class_ids, active_action_class_ids \
= KL.Lambda(parse_image_meta_graph,
mask=[None, None, None, None, None],
arguments={'config': config}
)(input_image_meta)
#_, _, _, active_class_ids \
# = KL.Lambda(lambda x: parse_image_meta_graph(x),
# mask=[None, None, None, None])(input_image_meta)
#active_actor_class_ids = active_class_ids[:, :config.NUM_ACTOR_CLASSES]
#active_action_class_ids = active_class_ids[:, config.NUM_ACTOR_CLASSES:]
if not config.USE_RPN_ROIS:
# Ignore predicted ROIs and use ROIs provided as an input.
input_rois = KL.Input(shape=[config.POST_NMS_ROIS_TRAINING, 4],
name="input_roi",
dtype=np.int32)
# Normalize coordinates to 0-1 range.
target_rois = KL.Lambda(lambda x: K.cast(x, tf.float32) /
image_scale[:4])(input_rois)
else:
target_rois = rpn_rois
# Generate detection targets
# Subsamples proposals and generates target outputs for training
# Note that proposals, gt_boxes, and gt_masks might be zero padded
# Equally, returned rois and targets might be zero padded as well
# TODO: delete tower arg
rois, target_actor_class_ids, target_action_class_ids, target_bbox, target_mask =\
DetectionTargetLayer(config, tower='rgb', name="proposal_targets")([
target_rois, gt_boxes, input_gt_masks])
# Network Heads
# TODO: verify that this handles zero padded ROIs
mrcnn_actor_class_logits, mrcnn_actor_probs, \
mrcnn_action_class_logits, mrcnn_action_probs, mrcnn_bbox = \
fpn_classifier_graph(rois, mrcnn_feature_maps, config.IMAGE_SHAPE,
config.POOL_SIZE, config.NUM_ACTOR_CLASSES, config.NUM_ACTION_CLASSES)
mrcnn_mask = build_fpn_mask_graph(rois, mrcnn_feature_maps,
config.IMAGE_SHAPE,
config.MASK_POOL_SIZE,
config.NUM_ACTOR_CLASSES)
# TODO: clean up (use tf.identify if necessary)
output_rois = KL.Lambda(lambda x: x * 1, name="output_rois")(rois)
# Losses
rpn_class_loss = KL.Lambda(lambda x: rpn_class_loss_graph(*x),
name="rpn_class_loss")(
[input_rpn_match, rpn_class_logits])
rpn_bbox_loss = KL.Lambda(
lambda x: rpn_bbox_loss_graph(config, *x),
name="rpn_bbox_loss")(
[input_rpn_bbox, input_rpn_match, rpn_bbox])
actor_class_loss = KL.Lambda(lambda x: mrcnn_class_loss_graph(*x),
name="mrcnn_actor_class_loss")([
target_actor_class_ids,
mrcnn_actor_class_logits,
active_actor_class_ids
])
action_class_loss = KL.Lambda(lambda x: mrcnn_class_loss_graph(*x),
name="mrcnn_action_class_loss")([
target_action_class_ids,
mrcnn_action_class_logits,
active_action_class_ids
])
bbox_loss = KL.Lambda(lambda x: mrcnn_bbox_loss_graph(*x),
name="mrcnn_bbox_loss")([
target_bbox, target_actor_class_ids,
mrcnn_bbox
])
mask_loss = KL.Lambda(lambda x: mrcnn_mask_loss_graph(*x),
name="mrcnn_mask_loss")([
target_mask, target_actor_class_ids,
mrcnn_mask
])
# Model
inputs = [
input_rgb_clip, input_flow_clip, input_image_meta,
input_labeled_frame_id, input_rpn_match, input_rpn_bbox,
input_gt_boxes, input_gt_masks
]
if not config.USE_RPN_ROIS:
inputs.append(input_rois)
outputs = [
rpn_class_logits, rpn_class, rpn_bbox,
mrcnn_actor_class_logits, mrcnn_actor_probs,
mrcnn_action_class_logits, mrcnn_action_probs, mrcnn_bbox,
mrcnn_mask, rpn_rois, output_rois, rpn_class_loss,
rpn_bbox_loss, actor_class_loss, action_class_loss, bbox_loss,
mask_loss
]
model = KM.Model(inputs, outputs, name='mask_rcnn')
else:
# Network Heads
# Proposal classifier and BBox regressor heads
mrcnn_actor_class_logits, mrcnn_actor_class, \
mrcnn_action_class_logits, mrcnn_action_class, mrcnn_bbox =\
fpn_classifier_graph(rpn_rois, mrcnn_feature_maps, config.IMAGE_SHAPE,
config.POOL_SIZE, config.NUM_ACTOR_CLASSES, config.NUM_ACTION_CLASSES)
# Detections
# output is [batch, num_detections, (y1, x1, y2, x2, actor_class_id, action_class_id, score)]
# in image coordinates
detections = DetectionLayer(config, name="mrcnn_detection")([
rpn_rois, mrcnn_actor_class, mrcnn_action_class, mrcnn_bbox,
input_image_meta
])
# Convert boxes to normalized coordinates
# TODO: let DetectionLayer return normalized coordinates to avoid
# unnecessary conversions
h, w = config.IMAGE_SHAPE[:2]
detection_boxes = KL.Lambda(
lambda x: x[..., :4] / np.array([h, w, h, w]))(detections)
# Create masks for detections
mrcnn_mask = build_fpn_mask_graph(detection_boxes,
mrcnn_feature_maps,
config.IMAGE_SHAPE,
config.MASK_POOL_SIZE,
config.NUM_ACTOR_CLASSES)
model = KM.Model([
input_rgb_clip, input_flow_clip, input_image_meta,
input_labeled_frame_id
], [
detections, mrcnn_actor_class, mrcnn_action_class, mrcnn_bbox,
mrcnn_mask, rpn_rois, rpn_class, rpn_bbox
],
name='mask_rcnn')
# Add multi-GPU support.
if config.GPU_COUNT > 1:
model = ParallelModel(model, config.GPU_COUNT)
return model
def __init__(self, shape):
self.re_rate = 0.9
self.inputs = layers.Input(shape=shape)
self.f_block = layers.Conv3D(4, (3, 3, 3), activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(self.inputs)
self.bn1 = layers.BatchNormalization()(self.f_block)
self.mp1 = layers.MaxPooling3D((2, 2, 2))(self.bn1)
self.f_block1 = layers.Conv3D(16, (3, 3, 3), activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(self.mp1)
# self.f_block1 = layers.Conv3D(32, (3, 3, 3), activation='relu',
# kernel_regularizer=regularizers.l2(self.re_rate),
# padding='same')(self.f_block1)
self.bn2 = layers.BatchNormalization()(self.f_block1)
self.mp2 = layers.MaxPooling3D((2, 2, 2))(self.f_block1)
self.f_block2 = layers.Conv3D(16, (3, 3, 3), activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(self.f_block1)
self.f_block2 = layers.BatchNormalization()(self.f_block2)
# self.f_block2 = layers.Conv3D(16, (3, 3, 3), activation='relu',
# kernel_regularizer=regularizers.l2(self.re_rate),
# padding='same')(self.f_block2)
# self.bn3 = layers.BatchNormalization()(self.f_block2)
self.mp3 = layers.MaxPooling3D((2, 2, 2))(self.f_block2)
self.b_back3 = layers.Conv3D(16, (3, 3, 3), activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(layers.UpSampling3D((2, 2, 2))(self.mp3))
self.b_back3 = layers.BatchNormalization()(self.b_back3)
# self.b_back3 = layers.Conv3D(16, (3, 3, 3), activation='relu',
# kernel_regularizer=regularizers.l2(self.re_rate),
# padding='same')(self.b_back3)
self.cat1 = layers.concatenate([self.b_back3, self.f_block2])
self.bn4 = layers.BatchNormalization()(self.cat1)
self.b_back4 = layers.Conv3D(16, (3, 3, 3), activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(self.bn4)
self.b_back4 = layers.BatchNormalization()(self.b_back4)
self.b_back4 = layers.Conv3D(16, (3, 3, 3), activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(self.b_back4)
self.b_back4 = layers.BatchNormalization()(self.b_back4)
self.b_back4 = layers.concatenate([self.b_back4, self.f_block1])
# two classify begin
# self.b_back4 = layers.MaxPooling3D((2, 2, 2))(self.b_back4)
# end
self.bn5 = layers.BatchNormalization()(self.b_back4)
self.b_back5 = layers.Conv3D(32, (2, 2, 2), activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(self.bn5)
self.b_back5 = layers.BatchNormalization()(self.b_back5)
self.b_back5 = layers.MaxPooling3D((2, 2, 2))(self.b_back5)
self.b_back6 = layers.Conv3D(64, (2, 2, 2), activation='relu',
kernel_regularizer=regularizers.l2(self.re_rate),
padding='same')(self.b_back5)
self.b_back6 = layers.BatchNormalization()(self.b_back6)
self.b_back6 = layers.MaxPooling3D((2, 2, 2))(self.b_back6)
# self.drop = layers.Dropout(rate=0.8)(self.b_back5)
self.transform = layers.Flatten()(self.b_back6)
# five classify num=32
self.dense1 = layers.Dense(64, activation='relu')(self.transform)
self.dense1 = layers.Dropout(rate=0.5)(self.dense1)
# five classify rate=0.3
# self.dense1 = layers.Dropout(rate=0.3)(self.dense1)
# self.dense1 = layers.Dense(16, activation='relu')(self.dense1)
self.dense2 = layers.Dense(32, activation='relu')(self.dense1)
# five classify begin
# self.dense2 = layers.Dense(8, activation='relu')(self.dense2)
# self.dense3 = layers.Dense(5, activation='softmax')(self.dense2)
self.dense3 = layers.Dense(4, activation='softmax')(self.dense2)
# five classify end
# two classify begin
# self.dense3 = layers.Dense(8, activation='relu')(self.dense2)
# self.dense3 = layers.Dense(1, activation='sigmoid')(self.dense2)
# two classify end
self.model = keras.Model(input=self.inputs, output=self.dense3)
