def hybrid_3pic(args, drop=0.2):
# ************************3d volume input******************************************************************
img_input = Input(batch_shape=(args.b, args.input_size, args.input_size, args.input_cols, 1), name='volumetric_data')
# ************************(batch*d3cols)*2dvolume--2D DenseNet branch**************************************
for b in range(args.b):
small_input = Lambda(divi_batch, arguments={'h1': b, 'h2': b+1})(img_input)
input2d = Lambda(slice, arguments={'h1': 0, 'h2': 2})(small_input)
single = Lambda(slice, arguments={'h1':0, 'h2':1})(small_input)
input2d = concatenate([single, input2d], axis=3)
for i in range(args.input_cols - 2):
input2d_tmp = Lambda(slice, arguments={'h1': i, 'h2': i + 3})(small_input)
input2d = concatenate([input2d, input2d_tmp], axis=0)
if i == args.input_cols - 3:
final1 = Lambda(slice, arguments={'h1': args.input_cols-2, 'h2': args.input_cols})(small_input)
final2 = Lambda(slice, arguments={'h1': args.input_cols-1, 'h2': args.input_cols})(small_input)
final = concatenate([final1, final2], axis=3)
input2d = concatenate([input2d, final], axis=0)
input2d = Lambda(slice_last)(input2d)
if b==0: a = input2d
else: a = concatenate([a, input2d], axis=0)
input2d = a
# ******************************stack to 3D volumes *******************************************************
feature2d, classifer2d = ResUNet_weight(input2d, trainable= False, bn_train=False)
# classifer2d = Activation('softmax')(classifer2d)
for b in range(args.b):
small_class2d = Lambda(divi_batch, arguments={'h1': b*args.input_cols, 'h2': (b+1)*args.input_cols})(classifer2d)
small_feature2d = Lambda(divi_batch, arguments={'h1': b*args.input_cols, 'h2': (b+1)*args.input_cols})(feature2d)
res2d = Lambda(slice2d, arguments={'h1': 0, 'h2': 1})(small_class2d)
fea2d = Lambda(slice2d, arguments={'h1':0, 'h2':1})(small_feature2d)
for j in range(args.input_cols - 1):
score = Lambda(slice2d, arguments={'h1': j + 1, 'h2': j + 2})(small_class2d)
fea2d_slice = Lambda(slice2d, arguments={'h1': j + 1, 'h2': j + 2})(small_feature2d)
res2d = concatenate([res2d, score], axis=3)
fea2d = concatenate([fea2d, fea2d_slice], axis=3)
if b==0:
r = res2d
f = fea2d
else:
r = concatenate([r, res2d], axis=0)
f = concatenate([f, fea2d], axis=0)
res2d = r
fea2d = f
# *************************** 3d DenseNet on 3D volume (concate with feature map )*********************************
res2d_input = Lambda(lambda x: x*250 )(res2d)
input3d_ori = Lambda(slice, arguments={'h1': 0, 'h2': args.input_cols})(img_input)
# res2d_input = Lambda(tumor_class)(res2d_input)
input3d = concatenate([input3d_ori, res2d_input], axis=4)
# feature3d, classifer3d = U_net3D_weight_32col(input3d)
input3d_1 = Conv3D(32, (3, 3, 3), padding="same")(input3d)
input3d_1 = BatchNormalization(axis=-1)(input3d_1)
input3d_1 = Activation('relu')(input3d_1)
input3d = concatenate([input3d, input3d_1], axis=4)
input3d = Conv3D(32, (3, 3, 3), padding="same")(input3d)
input3d = BatchNormalization(axis=-1)(input3d)
feature3d = Activation('relu')(input3d)
final = add([feature3d, fea2d])
# final = concatenate([feature3d, fea2d], axis=4)
final_conv = Conv3D(64, (3, 3, 3), padding="same", name='fianl_conv')(final)
if drop>0: final_conv = Dropout(rate=drop)(final_conv)
final_bn = BatchNormalization(name="final_bn")(final_conv)
final_ac = Activation('relu', name='final_ac')(final_bn)
classifer = Conv3D(3, (1, 1, 1), padding="same", name='2d3dclassifer')(final_ac)
classifer = Activation('softmax')(classifer)
model = Model( inputs = img_input,outputs = classifer, name='auto3d_residual_conv')
return model
def Conv3D_KDD():
WRF_inputs = Input(shape=(num_frames, 159, 159,
fea_dim)) # (bs, 6, 159, 159, fea_dim)
_history_inputs = Input(shape=(num_frames_truth, 159, 159,
1)) # (bs,3,159,159,1)
# history_inputs = Lambda(lambda x: K.squeeze(x, axis=-1))(_history_inputs) # (bs, 3, 159, 159)
history_inputs = Permute(
(4, 2, 3, 1))(_history_inputs) # (bs, 1, 159, 159, 3)
conv_1 = Conv3D(filters=128,
kernel_size=(2, 1, 1),
padding='same',
name='conv3d_1')(WRF_inputs)
conv_1 = Activation('relu')(conv_1)
conv_2 = Conv3D(filters=128,
kernel_size=(1, 3, 3),
padding='same',
name='conv3d_2')(conv_1)
conv_2 = Activation('relu')(conv_2)
conv_3 = Conv3D(filters=256,
kernel_size=(2, 3, 3),
padding='same',
name='conv3d_3')(conv_2)
conv_3 = Activation('relu')(conv_3)
conv_4 = Conv3D(filters=128,
kernel_size=(3, 1, 1),
padding='same',
name='conv3d_4')(conv_3)
conv_4 = Activation('relu')(conv_4)
conv_5 = Conv3D(filters=128,
kernel_size=(1, 3, 3),
padding='same',
name='conv3d_5')(conv_4)
conv_5 = Activation('relu')(conv_5)
conv_6 = Conv3D(filters=64,
kernel_size=(3, 3, 3),
padding='same',
name='conv3d_6')(conv_5)
conv_6 = Activation('relu')(conv_6)
steps = []
for i in range(num_frames):
conv_6_i = Cropping3D(cropping=((i, num_frames - i - 1), (0, 0),
(0,
0)))(conv_6) # (bs, 1, 159, 159, 64)
conv2d_in = concatenate([history_inputs, conv_6_i],
axis=-1) # (bs, 1, 159, 159, 64+3)
conv2d_in = Lambda(lambda x: K.squeeze(x, axis=1))(
conv2d_in) # (bs, 159, 159, 67)
conv2d_1_i = Conv2D(filters=64,
kernel_size=(7, 7),
padding='same',
name='conv2d_1_%d' % i)(conv2d_in)
conv2d_1_i = Activation('relu')(conv2d_1_i)
conv2d_2_i = Conv2D(filters=1,
kernel_size=(7, 7),
padding='same',
name='conv2d_2_%d' % i)(conv2d_1_i)
steps.append(conv2d_2_i)
conv_out = concatenate(steps, axis=1) # (bs, 6, 159, 159, 1)
outputs = Reshape((-1, 159 * 159, 1),
input_shape=(-1, 159, 159, 1))(conv_out)
return Model([WRF_inputs, _history_inputs], outputs, name='Conv3D-KDD')
def get_model(backend='tf', with_weights=''):
""" Return the Keras model of the network
"""
model = Sequential()
if backend == 'tf':
input_shape = (16, 112, 112, 3) # l, h, w, c
else:
input_shape = (3, 16, 112, 112) # c, l, h, w
model.add(
Conv3D(64,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv1',
input_shape=input_shape))
model.add(
MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
border_mode='valid',
name='pool1'))
# 2nd layer group
model.add(
Conv3D(128,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv2'))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool2'))
# 3rd layer group
model.add(
Conv3D(256,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv3a'))
model.add(
Conv3D(256,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv3b'))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool3'))
# 4th layer group
model.add(
Conv3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv4a'))
model.add(
Conv3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv4b'))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool4'))
# 5th layer group
model.add(
Conv3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv5a'))
model.add(
Conv3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv5b'))
model.add(ZeroPadding3D(padding=((0, 0), (0, 1), (0, 1)), name='zeropad5'))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool5'))
model.add(Flatten())
# FC layers group
model.add(Dense(4096, activation='relu', name='fc6'))
model.add(Dropout(.5))
model.add(Dense(4096, activation='relu', name='fc7'))
model.add(Dropout(.5))
model.add(Dense(487, activation='softmax', name='fc8'))
if with_weights == "":
model.load_weights(WEIGHTS_PATH)
return model
def big_assemble_models(self):
image = Input(shape=( 25, 25, 25,1 ), name='image')
x = _Conv3D(32, 5, 5,5,border_mode='same')(image)
x = LeakyReLU()(x)
x = Dropout(discr_drop_out)(x)
x = ZeroPadding3D((2, 2,2))(x)
x = _Conv3D(8, 5, 5,5,border_mode='valid')(x)
x = LeakyReLU()(x)
x = BatchNormalization()(x)
x = Dropout(discr_drop_out)(x)
x = ZeroPadding3D((2, 2, 2))(x)
x = Conv3D(8, 5, 5,5,border_mode='valid')(x)
x = LeakyReLU()(x)
x = BatchNormalization()(x)
x = Dropout(discr_drop_out)(x)
x = ZeroPadding3D((1, 1, 1))(x)
x = Conv3D(8, 5, 5,5,border_mode='valid')(x)
x = LeakyReLU()(x)
x = BatchNormalization()(x)
x = Dropout(discr_drop_out)(x)
x = AveragePooling3D((2, 2, 2))(x)
h = Flatten()(x)
fake = Dense(1, activation='sigmoid', name='generation')(h)
aux = Dense(1, activation='linear', name='auxiliary')(h)
ecal = Lambda(lambda x: K.sum(x, axis=(1, 2, 3)), name='sum_cell')(image)
self.discriminator = Model(output=[fake, aux, ecal], input=image, name='discriminator_model')
latent = Input(shape=(self.latent_size, ))
x = Dense(64 * 7* 7, init='he_uniform')(latent)
x = Reshape((7, 7,8, 8))(x)
x = Conv3D(64, 6, 6, 8, border_mode='same', init='he_uniform' )(x)
x = LeakyReLU()(x)
x = BatchNormalization()(x)
x = UpSampling3D(size=(2, 2, 2))(x)
x = ZeroPadding3D((2, 2, 0))(x)
x = Conv3D(6, 6, 5, 8, init='he_uniform')(x)
x = LeakyReLU()(x)
x = BatchNormalization()(x)
x = UpSampling3D(size=(2, 2, 3))(x)
x = ZeroPadding3D((1,0,3))(x)
x = Conv3D(6, 3, 3, 8, init='he_uniform')(x)
x = LeakyReLU()(x)
x = Conv3D(1, 2, 2, 2, bias=False, init='glorot_normal')(x)
x = Activation('relu')(x)
loc = Model(latent, x)
#fake_image = loc(latent)
self.generator = Model(input=latent, output=x, name='generator_model')
c_fake, c_aux, c_ecal = self.discriminator(x)
self.combined = Model(
input = latent,
output = [c_fake, c_aux, c_ecal],
name='combined_model'
)
def build_discriminator(self):
def d_layer(layer_input,
filters,
kernel_size=(4, 4, 2),
strides=(2, 2, 2),
bn=True):
"""Discriminator layer"""
init = RandomNormal(stddev=0.02)
d = Conv3D(filters,
kernel_size=kernel_size,
strides=strides,
padding='same',
kernel_initializer=init)(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if bn:
#d = BatchNormalization(momentum=0.8)(d)
d = InstanceNormalization()(d)
return d
img_A = Input(shape=self.img_shape)
img_B = Input(shape=self.img_shape)
# Concatenate image and conditioning image by channels to produce input
combined_imgs = Concatenate(axis=-1)([img_A, img_B])
## testing
#combined_imgs = GaussianNoise(0.02)(combined_imgs)
if self.coordconv:
combined_imgs = CoordinateChannel3D()(combined_imgs)
n_d_layer = 4
d_layers = []
df = self.df
dout = self.depth
for i in range(n_d_layer):
if i == 0:
d_layers.append(
d_layer(combined_imgs,
df,
kernel_size=(4, 4, 2),
strides=(2, 2, 2)))
else:
z = 2
s = 2
if dout == 2:
z = 1
if i == n_d_layer - 1:
s = 1
df = min(df * 2, self.df * 8)
d_layers.append(
d_layer(d_layers[-1],
df,
kernel_size=(4, 4, z),
strides=(s, s, z)))
dout = max(2, int(0.5 * dout))
init = RandomNormal(stddev=0.02)
features = Conv3D(1,
kernel_size=(4, 4, 1),
strides=1,
padding='same',
kernel_initializer=init)(d_layers[-1])
validity = Activation('sigmoid')(features)
return Model([img_A, img_B], validity), Model([img_A, img_B], features)
def build(input_shape, num_outputs):
print('original input shape:', input_shape)
_handle_dim_ordering()
if len(input_shape) != 4:
raise Exception(
"Input shape should be a tuple (nb_channels, kernel_dim1, kernel_dim2, kernel_dim3)"
)
print('original input shape:', input_shape)
# orignal input shape: 1,7,7,200
if K.image_dim_ordering() == 'tf':
input_shape = (input_shape[1], input_shape[2], input_shape[3],
input_shape[0])
print('change input shape:', input_shape)
# 用keras中函数式模型API，不用序贯模型API
# 张量流输入
input = Input(shape=input_shape)
# print(input.shape)
# ( ?,7,7,200,1 )
conv1 = Conv3D(filters=15,
kernel_size=(15, 15, 24),
strides=(12, 12, 12),
kernel_regularizer=regularizers.l2(0.01))(input)
print(conv1.shape)
# ( None, 1,1,15,15 )
# 这一步整一个 3*3 空间的，然后在 reshape 不就好了
l = Reshape((15, 15, 1))(conv1)
print(l)
# l = Conv2D(filters=64, kernel_size=(3, 3), strides=1, kernel_regularizer=regularizers.l2(0.01))(conv2_spc)
# print(l)
# print(l.shape)
# ( ?,20,60,1 )
# conv11
l = Conv2D(32, (3, 3), padding='same', name='conv11',
trainable=False)(l)
l = Activation('relu', name='conv11_relu')(l)
l = BatchNormalization(name='conv11_bn')(l)
# conv12
l_offset = ConvOffset2D(32, name='conv12_offset')(l)
l = Conv2D(64, (3, 3),
padding='same',
strides=(2, 2),
name='conv12',
trainable=False)(l_offset)
l = Activation('relu', name='conv12_relu')(l)
l = BatchNormalization(name='conv12_bn')(l)
# conv21
l_offset = ConvOffset2D(64, name='conv21_offset')(l)
l = Conv2D(64, (3, 3),
padding='same',
strides=(2, 2),
name='conv21',
trainable=False)(l_offset)
l = Activation('relu', name='conv21_relu')(l)
l = BatchNormalization(name='conv21_bn')(l)
# conv22
# l_offset = ConvOffset2D(64, name='conv22_offset')(l)
# l = Conv2D(64, (3, 3), padding='same', strides=(2, 2), name='conv22', trainable=False)(l_offset)
# l = Activation('relu', name='conv22_relu')(l)
# l = BatchNormalization(name='conv22_bn')(l)
# out
l = GlobalAvgPool2D(name='avg_pool')(l)
# 输入分类器
# Classifier block
dense = Dense(units=num_outputs,
activation="softmax",
kernel_initializer="he_normal")(l)
model = Model(inputs=input, outputs=dense)
return model
def c3d(self):
"""
Build a 3D convolutional network, aka C3D.
https://arxiv.org/pdf/1412.0767.pdf
With thanks:
https://gist.github.com/albertomontesg/d8b21a179c1e6cca0480ebdf292c34d2
"""
model = Sequential()
# 1st layer group
model.add(
Conv3D(64,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv1',
subsample=(1, 1, 1),
input_shape=self.input_shape))
model.add(
MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
border_mode='valid',
name='pool1'))
# 2nd layer group
model.add(
Conv3D(128,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv2',
subsample=(1, 1, 1)))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool2'))
# 3rd layer group
model.add(
Conv3D(256,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv3a',
subsample=(1, 1, 1)))
model.add(
Conv3D(256,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv3b',
subsample=(1, 1, 1)))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool3'))
# 4th layer group
model.add(
Conv3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv4a',
subsample=(1, 1, 1)))
model.add(
Conv3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv4b',
subsample=(1, 1, 1)))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool4'))
# 5th layer group
model.add(
Conv3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv5a',
subsample=(1, 1, 1)))
model.add(
Conv3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv5b',
subsample=(1, 1, 1)))
model.add(ZeroPadding3D(padding=(0, 1, 1)))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool5'))
model.add(Flatten())
# FC layers group
model.add(Dense(4096, activation='relu', name='fc6'))
model.add(Dropout(0.5))
model.add(Dense(4096, activation='relu', name='fc7'))
model.add(Dropout(0.5))
model.add(Dense(self.nb_classes, activation='softmax'))
return model
def f(input):
activation = _bn_relu(input)
return Conv3D(filters=filters, kernel_size=kernel_size,
strides=strides, kernel_initializer=kernel_initializer,
padding=padding,
kernel_regularizer=kernel_regularizer)(activation)
def model():
input_1 = Input(shape=(7, 7, 200))
input_2 = Input(shape=(27, 27, 30))
# BAM
CAB_conv1 = Conv2D(16, (3, 3), padding='same', strides=(1, 1),
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4),
# use_bias=False
)(input_1)
CAB_bn1 = BatchNormalization()(CAB_conv1)
CAB_relu1 = PReLU()(CAB_bn1)
CAB_avg_pool1 = AveragePooling2D()(CAB_relu1)
CAB_conv4 = Conv2D(32, (3, 3), padding='same', strides=(1, 1),
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4),
# use_bias=False
)(CAB_avg_pool1)
CAB_bn4 = BatchNormalization()(CAB_conv4)
CAB_relu4 = PReLU()(CAB_bn4)
CAB_conv5 = Conv2D(32, (3, 3), padding='same', strides=(1, 1),
kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4),
# use_bias=False
)(CAB_relu4)
CAB_bn5 = BatchNormalization()(CAB_conv5)
CAB_relu5 = PReLU()(CAB_bn5)
CAB_global_pool = GlobalAveragePooling2D()(CAB_relu5)
# ===================================================================================================================
CAB_reshape = Reshape((1, CAB_global_pool._keras_shape[1]))(CAB_global_pool)
CAB_conv6 = Conv1D(50, (32), padding='same', strides=(1), kernel_initializer='glorot_uniform', use_bias=False)(
CAB_reshape)
CAB_bn6 = BatchNormalization()(CAB_conv6)
CAB_relu6 = PReLU()(CAB_bn6)
CAB_conv7 = Conv1D(200, (50), padding='same', strides=(1), kernel_initializer='glorot_uniform', use_bias=False)(
CAB_relu6)
CAB_bn7 = BatchNormalization()(CAB_conv7)
CAB_sigmoid = Activation('sigmoid')(CAB_bn7)
# ==================================================================================================================
CAB_mul = multiply([input_1, CAB_sigmoid])
input_spe = Reshape((CAB_mul._keras_shape[1], CAB_mul._keras_shape[2], CAB_mul._keras_shape[3], 1))(CAB_mul)
# input_spe = Reshape((input_1._keras_shape[1], input_1._keras_shape[2], input_1._keras_shape[3], 1))(input_1)
conv_spe1 = Conv3D(32, (1, 1, 7), padding='valid', strides=(1, 1, 2))(input_spe)
bn_spe1 = BatchNormalization()(conv_spe1)
relu_spe1 = PReLU()(bn_spe1)
conv_spe11 = Conv3D(num_filters_spe, (1, 1, 7), padding='same', strides=(1, 1, 1))(relu_spe1)
bn_spe11 = BatchNormalization()(conv_spe11)
relu_spe11 = PReLU()(bn_spe11)
blockconv_spe1 = Conv3D(num_filters_spe, (1, 1, 7), padding='same', strides=(1, 1, 1))(relu_spe11)
blockbn_spe1 = BatchNormalization()(blockconv_spe1)
blockrelu_spe1 = PReLU()(blockbn_spe1)
conv_spe2 = Conv3D(num_filters_spe, (1, 1, 7), padding='same', strides=(1, 1, 1))(blockrelu_spe1)
add_spe1 = add([relu_spe11, conv_spe2])
bn_spe2 = BatchNormalization()(add_spe1)
relu_spe2 = PReLU()(bn_spe2)
blockconv_spe2 = Conv3D(num_filters_spe, (1, 1, 7), padding='same', strides=(1, 1, 1))(relu_spe2)
blockbn_spe2 = BatchNormalization()(blockconv_spe2)
blockrelu_spe2 = PReLU()(blockbn_spe2)
conv_spe4 = Conv3D(num_filters_spe, (1, 1, 7), padding='same', strides=(1, 1, 1))(blockrelu_spe2)
add_spe2 = add([relu_spe2, conv_spe4])
bn_spe4 = BatchNormalization()(add_spe2)
relu_spe4 = PReLU()(bn_spe4)
blockconv_spe3 = Conv3D(num_filters_spe, (1, 1, 7), padding='same', strides=(1, 1, 1))(relu_spe4)
blockbn_spe3 = BatchNormalization()(blockconv_spe3)
blockrelu_spe3 = PReLU()(blockbn_spe3)
conv_spe41 = Conv3D(num_filters_spe, (1, 1, 7), padding='same', strides=(1, 1, 1))(blockrelu_spe3)
add_spe3 = add([relu_spe4, conv_spe41])
bn_spe41 = BatchNormalization()(add_spe3)
relu_spe41 = PReLU()(bn_spe41)
add_all_spe = add([relu_spe2, relu_spe4, relu_spe41])
conv_spe6 = Conv3D(8, (1, 1, 97), padding='valid', strides=(1, 1, 1))(add_all_spe)
bn_spe6 = BatchNormalization()(conv_spe6)
relu_spe6 = PReLU()(bn_spe6)
input_spa = Reshape((input_2._keras_shape[1], input_2._keras_shape[2], input_2._keras_shape[3], 1))(input_2)
conv_spa1 = Conv3D(16, (5, 5, 30), padding='valid', strides=(1, 1, 1))(input_spa)
print('conv_spa1 shape:', conv_spa1.shape)
bn_spa1 = BatchNormalization()(conv_spa1)
relu_spa1 = PReLU()(bn_spa1)
reshape_spa1 = Reshape((relu_spa1._keras_shape[1], relu_spa1._keras_shape[2], relu_spa1._keras_shape[4], relu_spa1._keras_shape[3]))(relu_spa1)
conv_spa11 = Conv3D(num_filters_spa, (3,3,1), padding='same', strides=(1, 1, 1))(reshape_spa1)
bn_spa11 = BatchNormalization()(conv_spa11)
relu_spa11 = PReLU()(bn_spa11)
VIS_conv1 = Conv3D(16, (1, 1, 16), padding='valid', strides=(1, 1, 1),kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(relu_spa11)
VIS_BN1 = BatchNormalization()(VIS_conv1)
VIS_relu1 = Activation('relu')(VIS_BN1)
VIS_SHAPE1 = Reshape((VIS_relu1._keras_shape[1] * VIS_relu1._keras_shape[2], VIS_relu1._keras_shape[4]))(
VIS_relu1)
VIS_conv2 = Conv3D(16, (1, 1, 16), padding='valid', strides=(1, 1, 1),kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(relu_spa11)
VIS_BN2 = BatchNormalization()(VIS_conv2)
VIS_relu2 = Activation('relu')(VIS_BN2)
VIS_SHAPE2 = Reshape((VIS_relu2._keras_shape[1] * VIS_relu2._keras_shape[2], VIS_relu2._keras_shape[4]))(
VIS_relu2)
trans_VIS_SHAPE2 = Permute((2, 1))(VIS_SHAPE2)
VIS_conv3 = Conv3D(16, (1, 1, 16), padding='valid', strides=(1, 1, 1),kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(relu_spa11)
VIS_BN3 = BatchNormalization()(VIS_conv3)
VIS_relu3 = Activation('relu')(VIS_BN3)
VIS_SHAPE3 = Reshape((VIS_relu3._keras_shape[1] * VIS_relu3._keras_shape[2], VIS_relu3._keras_shape[4]))(
VIS_relu3)
VIS_mul1 = dot([VIS_SHAPE1, trans_VIS_SHAPE2], axes=(2, 1))
VIS_sigmoid = Activation('sigmoid')(VIS_mul1)
VIS_mul2 = dot([VIS_sigmoid, VIS_SHAPE3], axes=(2, 1))
VIS_SHAPEall = Reshape((23, 23, 16, 1))(VIS_mul2)
VIS_conv4 = Conv3D(16, (16, 1, 1), padding='same', strides=(1),kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(VIS_SHAPEall)
VIS_BN4 = BatchNormalization()(VIS_conv4)
VIS_ADD = add([relu_spa11, VIS_BN4])
blockconv_spa1 = Conv3D(num_filters_spa, (3, 3, 1), padding='same', strides=(1, 1, 1))(VIS_ADD)
blockbn_spa1 = BatchNormalization()(blockconv_spa1)
blockrelu_spa1 = PReLU()(blockbn_spa1)
conv_spa2 = Conv3D(num_filters_spa, (3, 3, 1), padding='same', strides=(1))(blockrelu_spa1)
add_spa1 = add([VIS_ADD, conv_spa2])
bn_spa2 = BatchNormalization()(add_spa1)
relu_spa2 = PReLU()(bn_spa2)
blockconv_spa2 = Conv3D(num_filters_spa, (3, 3, 1), padding='same', strides=(1))(relu_spa2)
blockbn_spa2 = BatchNormalization()(blockconv_spa2)
blockrelu_spa2 = PReLU()(blockbn_spa2)
conv_spa4 = Conv3D(num_filters_spa, (3, 3, 1), padding='same', strides=(1))(blockrelu_spa2)
add_spa2 = add([relu_spa2, conv_spa4])
bn_spa4 = BatchNormalization()(add_spa2)
relu_spa4 = PReLU()(bn_spa4)
blockconv_spa3 = Conv3D(num_filters_spa, (3, 3, 1), padding='same', strides=(1))(relu_spa4)
blockbn_spa3 = BatchNormalization()(blockconv_spa3)
blockrelu_spa3 = PReLU()(blockbn_spa3)
conv_spa41 = Conv3D(num_filters_spa, (3, 3, 1), padding='same', strides=(1))(blockrelu_spa3)
add_spa3 = add([relu_spa4, conv_spa41])
bn_spa41 = BatchNormalization()(add_spa3)
relu_spa41 = PReLU()(bn_spa41)
add_all_spa = add([relu_spa2, relu_spa4, relu_spa41])
conv_spa6 = Conv3D(num_filters_spa, (5, 5, 1), padding='valid', strides=(1, 1, 1))(add_all_spa)
bn_spa6 = BatchNormalization()(conv_spa6)
relu_spa6 = PReLU()(bn_spa6)
conv_spa7 = Conv3D(num_filters_spa, (5, 5, 1), padding='valid', strides=(1, 1, 1))(relu_spa6)
bn_spa7 = BatchNormalization()(conv_spa7)
relu_spa7 = PReLU()(bn_spa7)
conv_spa8 = Conv3D(num_filters_spa, (5, 5, 1), padding='valid', strides=(1, 1, 1))(relu_spa7)
bn_spa8 = BatchNormalization()(conv_spa8)
relu_spa8 = PReLU()(bn_spa8)
conv_spa81 = Conv3D(num_filters_spa, (5, 5, 1), padding='valid', strides=(1, 1, 1))(relu_spa8)
bn_spa81 = BatchNormalization()(conv_spa81)
relu_spa81 = PReLU()(bn_spa81)
conv_spa9 = Conv3D(8, (1, 1, 16), padding='valid', strides=(1, 1, 1))(relu_spa81)
bn_spa9 = BatchNormalization()(conv_spa9)
relu_spa9 = PReLU()(bn_spa9)
feature_fusion = concatenate([relu_spe6, relu_spa9])
reshape_all = Reshape((feature_fusion._keras_shape[1], feature_fusion._keras_shape[2],
feature_fusion._keras_shape[4], feature_fusion._keras_shape[3]))(
feature_fusion)
conv_all1 = Conv3D(16, (3,3,3), padding='same', strides=(1, 1, 1))(reshape_all)
print('convall1 shape:', conv_all1.shape)
bn_all1 = BatchNormalization()(conv_all1)
relu_all1 = PReLU()(bn_all1)
VIS_conv11 = Conv3D(16, (1, 1, 16), padding='valid', strides=(1, 1, 1), kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(relu_all1)
VIS_BN11 = BatchNormalization()(VIS_conv11)
VIS_relu11 = Activation('relu')(VIS_BN11)
VIS_SHAPE11 = Reshape((VIS_relu11._keras_shape[1] * VIS_relu11._keras_shape[2], VIS_relu11._keras_shape[4]))(
VIS_relu11)
VIS_conv21 = Conv3D(16, (1, 1, 16), padding='valid', strides=(1, 1, 1), kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(relu_all1)
VIS_BN21 = BatchNormalization()(VIS_conv21)
VIS_relu21 = Activation('relu')(VIS_BN21)
VIS_SHAPE21 = Reshape((VIS_relu21._keras_shape[1] * VIS_relu21._keras_shape[2], VIS_relu21._keras_shape[4]))(
VIS_relu21)
trans_VIS_SHAPE21 = Permute((2, 1))(VIS_SHAPE21)
VIS_conv31 = Conv3D(16, (1, 1, 16), padding='valid', strides=(1, 1, 1), kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(relu_all1)
VIS_BN31 = BatchNormalization()(VIS_conv31)
VIS_relu31 = Activation('relu')(VIS_BN31)
VIS_SHAPE31 = Reshape((VIS_relu31._keras_shape[1] * VIS_relu31._keras_shape[2], VIS_relu31._keras_shape[4]))(
VIS_relu31)
VIS_mul11 = dot([VIS_SHAPE11, trans_VIS_SHAPE21], axes=(2, 1))
VIS_sigmoid1 = Activation('sigmoid')(VIS_mul11)
VIS_mul21 = dot([VIS_sigmoid1, VIS_SHAPE31], axes=(2, 1))
VIS_SHAPEall1 = Reshape((7, 7, 16, 1))(VIS_mul21)
VIS_conv41 = Conv3D(16, (16, 1, 1), padding='same', strides=(1), kernel_initializer='glorot_uniform',
kernel_regularizer=l2(1e-4))(VIS_SHAPEall1)
VIS_BN41 = BatchNormalization()(VIS_conv41)
VIS_ADD1 = add([relu_all1, VIS_BN41])
conv_all2 = Conv3D(16, (3), padding='valid', strides=(1, 1, 1))(VIS_ADD1)
bn_all2 = BatchNormalization()(conv_all2)
relu_all2 = PReLU()(bn_all2)
flatten = Flatten()(relu_all2)
# drop0 = Dropout(0.5)(flatten)
dense = Dense(units=512, activation="relu", kernel_initializer="he_normal")(flatten)
# dense_1 = Dense(units=512, activation="relu", kernel_initializer="he_normal")(dense)
drop = Dropout(0.5)(dense)
dense_2 = Dense(units=256, activation="relu", kernel_initializer="he_normal")(drop)
drop1 = Dropout(0.5)(dense_2)
dense_3 = Dense(units=nb_classes, activation="softmax", kernel_initializer="he_normal")(drop1)
model = Model(inputs=[input_1, input_2], outputs=dense_3)
sgd = keras.optimizers.sgd(lr=0.001, momentum=0.9)
# adam = Adam(lr=0.001)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
model.summary()
return model
def create_model( self ):
print( "Build U-Net model" )
# Build U-Net model
inputs = Input( ( self.IMG_SIZE, self.IMG_SIZE, self.IMG_SIZE, 1 ) )
s = Lambda(lambda x: x / 255) (inputs)
c1 = Conv3D(16, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (s)
c1 = Dropout(0.1) (c1)
c1 = Conv3D(16, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c1)
p1 = MaxPooling3D((2, 2, 2)) (c1)
c2 = Conv3D(32, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (p1)
c2 = Dropout(0.1) (c2)
c2 = Conv3D(32, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c2)
p2 = MaxPooling3D((2, 2, 2)) (c2)
c3 = Conv3D(64, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (p2)
c3 = Dropout(0.2) (c3)
c3 = Conv3D(64, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c3)
p3 = MaxPooling3D((2, 2, 2)) (c3)
c4 = Conv3D(128, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (p3)
c4 = Dropout(0.2) (c4)
c4 = Conv3D(128, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c4)
p4 = MaxPooling3D(pool_size=(2, 2, 2)) (c4)
c5 = Conv3D(256, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (p4)
c5 = Dropout(0.3) (c5)
c5 = Conv3D(256, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c5)
u6 = Conv3DTranspose(128, (2, 2, 2), strides=(2, 2, 2), padding='same') (c5)
u6 = concatenate([u6, c4])
c6 = Conv3D(128, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u6)
c6 = Dropout(0.2) (c6)
c6 = Conv3D(128, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c6)
u7 = Conv3DTranspose(64, (2, 2, 2), strides=(2, 2, 2), padding='same') (c6)
u7 = concatenate([u7, c3])
c7 = Conv3D(64, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u7)
c7 = Dropout(0.2) (c7)
c7 = Conv3D(64, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c7)
u8 = Conv3DTranspose(32, (2, 2, 2), strides=(2, 2, 2), padding='same') (c7)
u8 = concatenate([u8, c2])
c8 = Conv3D(32, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u8)
c8 = Dropout(0.1) (c8)
c8 = Conv3D(32, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c8)
u9 = Conv3DTranspose(8, (2, 2, 2), strides=(2, 2, 2), padding='same') (c8)
u9 = concatenate([u9, c1], axis=-1)
c9 = Conv3D(16, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (u9)
c9 = Dropout(0.1) (c9)
c9 = Conv3D(16, (3, 3, 3), activation='elu', kernel_initializer='he_normal', padding='same') (c9)
outputs = Conv3D(self.NUM_CLASSES, (1, 1, 1), activation='softmax') (c9)
self.model = Model(inputs=[inputs], outputs=[outputs])
#self.model.compile(optimizer='adam', loss=iou_coef_loss, metrics=[ iou_coef_loss, dice_coef_loss, "accuracy" ])
self.model.compile(optimizer='adam', loss="categorical_crossentropy", metrics=[ iou_coef, dice_coef, iou_coef_loss, dice_coef_loss, "accuracy" ] )
self.model.summary()
def f(input):
conv = Conv3D(filters=filters, kernel_size=kernel_size,
strides=strides, kernel_initializer=kernel_initializer,
padding=padding,
kernel_regularizer=kernel_regularizer)(input)
return _bn_relu(conv)
conv_3D = [5, 5] # 卷积核尺寸
pool_3D = [2, 3] # 池化层尺寸
# 加载模型
model_exists = os.path.exists('model.h5')
if (model_exists):
model = load_model('model.h5')
print("**************************************************")
print("model.h5 model loaded")
else:
model = Sequential()
model.add(
Conv3D(filters_3D[0], (conv_3D[0], conv_3D[0], conv_3D[0]),
input_shape=(img_frames, img_rows, img_cols, 3),
activation='relu'))
model.add(
MaxPooling3D(pool_size=(pool_3D[0], pool_3D[0], pool_3D[0]),
data_format="channels_last"))
model.add(
Conv3D(filters_3D[1], (conv_3D[0], conv_3D[0], conv_3D[0]),
activation='relu'))
model.add(
MaxPooling3D(pool_size=(pool_3D[1], pool_3D[1], pool_3D[1]),
data_format="channels_last"))
model.add(Dropout(0.5))
def UNETGenerator(input_img_dim, num_output_channels):
"""
Creates the generator according to the specs in the paper below.
It's basically a skip layer AutoEncoder
Generator does the following:
1. Takes in an image
2. Generates an image from this image
Differs from a standard GAN because the image isn't random.
This model tries to learn a mapping from a suboptimal image to an optimal image.
[https://arxiv.org/pdf/1611.07004v1.pdf][5. Appendix]
:param input_img_dim: (channel, height, width, depth)
:param output_img_dim: (channel, height, width, depth)
:return:
"""
# -------------------------------
# ENCODER
# C64-C128-C256-C512-C512-C512-C512-C512
# 1 layer block = Conv - BN - LeakyRelu
# -------------------------------
stride = 2
merge_mode = 'concat'
# batch norm mode
bn_mode = 2
# batch norm merge axis
bn_axis = 1
# filter_sizes = [32*2, 64*2, 128*2, 256*2, 256*2, 256*2, 256*2, 256*2]
# filter_sizes = [32, 64, 128, 256, 256, 256, 256, 256]
filter_sizes = [
32 / 4, 64 / 4, 128 / 4, 256 / 4, 256 / 4, 256 / 4, 256 / 4, 256 / 4
]
input_layer = Input(shape=input_img_dim, name="unet_input")
# 1 encoder C64
# skip batchnorm on this layer on purpose (from paper)
en_1 = Conv3D(filters=filter_sizes[0],
kernel_size=(4, 4, 4),
padding='same',
strides=(stride, stride, stride))(input_layer)
en_1 = LeakyReLU(alpha=0.2)(en_1)
# 2 encoder C128
en_2 = Conv3D(filters=filter_sizes[1],
kernel_size=(4, 4, 4),
padding='same',
strides=(stride, stride, stride))(en_1)
en_2 = BatchNormalization(name='gen_en_bn_2', axis=bn_axis)(en_2)
en_2 = LeakyReLU(alpha=0.2)(en_2)
# 3 encoder C256
# en_3 = Conv3D(filters=filter_sizes[2], kernel_size=(4,4,4), padding='same', strides=(stride,stride,stride))(en_2)
# en_3 = BatchNormalization(name='gen_en_bn_3', axis=bn_axis)(en_3)
# en_3 = LeakyReLU(alpha=0.2)(en_3)
# 4 encoder C512
en_4 = Conv3D(filters=filter_sizes[3],
kernel_size=(4, 4, 4),
padding='same',
strides=(stride, stride, stride))(en_2)
en_4 = BatchNormalization(name='gen_en_bn_4', axis=bn_axis)(en_4)
en_4 = LeakyReLU(alpha=0.2)(en_4)
# 5 encoder C512
en_5 = Conv3D(filters=filter_sizes[4],
kernel_size=(4, 4, 4),
padding='same',
strides=(stride, stride, stride))(en_4)
en_5 = BatchNormalization(name='gen_en_bn_5', axis=bn_axis)(en_5)
en_5 = LeakyReLU(alpha=0.2)(en_5)
# 6 encoder C512
en_6 = Conv3D(filters=filter_sizes[5],
kernel_size=(4, 4, 4),
padding='same',
strides=(stride, stride, stride))(en_5)
en_6 = BatchNormalization(name='gen_en_bn_6', axis=bn_axis)(en_6)
en_6 = LeakyReLU(alpha=0.2)(en_6)
# 7 encoder C512
en_7 = Conv3D(filters=filter_sizes[6],
kernel_size=(4, 4, 4),
padding='same',
strides=(stride, stride, stride))(en_6)
en_7 = BatchNormalization(name='gen_en_bn_7', axis=bn_axis)(en_7)
en_7 = LeakyReLU(alpha=0.2)(en_7)
# 8 encoder C512
en_8 = Conv3D(filters=filter_sizes[7],
kernel_size=(4, 4, 4),
padding='same',
strides=(stride, stride, stride))(en_7)
en_8 = BatchNormalization(name='gen_en_bn_8', axis=bn_axis)(en_8)
en_8 = LeakyReLU(alpha=0.2)(en_8)
# -------------------------------
# DECODER
# CD512-CD1024-CD1024-C1024-C1024-C512-C256-C128
# filter_sizes = [256*2, 512*2, 512*2, 512*2, 512*2, 256*2, 128*2]
# filter_sizes = [256, 512, 512, 512, 512, 256, 128]
filter_sizes = [
256 / 4, 512 / 4, 512 / 4, 512 / 4, 512 / 4, 256 / 4, 128 / 4
]
# 1 layer block = Conv - Upsample - BN - DO - Relu
# also adds skip connections (merge). Takes input from previous layer matching encoder layer
# -------------------------------
# 1 decoder CD512 (decodes en_8)
de_1 = UpSampling3D(size=(2, 2, 2))(en_8)
de_1 = Conv3D(filters=filter_sizes[0],
kernel_size=(4, 4, 4),
padding='same')(de_1)
de_1 = BatchNormalization(name='gen_de_bn_1', axis=bn_axis)(de_1)
de_1 = Dropout(rate=0.5)(de_1)
# de_1 = merge([de_1, en_7], mode=merge_mode, concat_axis=1)
de_1 = concatenate([de_1, en_7], axis=1)
de_1 = Activation('relu')(de_1)
# 2 decoder CD1024 (decodes en_7)
de_2 = UpSampling3D(size=(2, 2, 2))(de_1)
de_2 = Conv3D(filters=filter_sizes[1],
kernel_size=(4, 4, 4),
padding='same')(de_2)
de_2 = BatchNormalization(name='gen_de_bn_2', axis=bn_axis)(de_2)
de_2 = Dropout(rate=0.5)(de_2)
# de_2 = merge([de_2, en_6], mode=merge_mode, concat_axis=1)
de_2 = concatenate([de_2, en_6], axis=1)
de_2 = Activation('relu')(de_2)
# 3 decoder CD1024 (decodes en_6)
de_3 = UpSampling3D(size=(2, 2, 2))(de_2)
de_3 = Conv3D(filters=filter_sizes[2],
kernel_size=(4, 4, 4),
padding='same')(de_3)
de_3 = BatchNormalization(name='gen_de_bn_3', axis=bn_axis)(de_3)
de_3 = Dropout(rate=0.5)(de_3)
# de_3 = merge([de_3, en_5], mode=merge_mode, concat_axis=1)
de_2 = concatenate([de_3, en_5], axis=1)
de_3 = Activation('relu')(de_3)
# 4 decoder CD1024 (decodes en_5)
de_4 = UpSampling3D(size=(2, 2, 2))(de_3)
de_4 = Conv3D(filters=filter_sizes[3],
kernel_size=(4, 4, 4),
padding='same')(de_4)
de_4 = BatchNormalization(name='gen_de_bn_4', axis=bn_axis)(de_4)
de_4 = Dropout(rate=0.5)(de_4)
# de_4 = merge([de_4, en_4], mode=merge_mode, concat_axis=1)
de_4 = concatenate([de_4, en_4], axis=1)
de_4 = Activation('relu')(de_4)
# 5 decoder CD1024 (decodes en_4)
# de_5 = UpSampling3D(size=(2,2,2))(de_4)
# de_5 = Conv3D(filters=filter_sizes[4], kernel_size=(4,4,4), padding='same')(de_5)
# de_5 = BatchNormalization(name='gen_de_bn_5', axis=bn_axis)(de_5)
# de_5 = Dropout(rate=0.5)(de_5)
# # de_5 = merge([de_5, en_3], mode=merge_mode, concat_axis=1)
# de_5 = concatenate([de_5, en_3], axis=1)
# de_5 = Activation('relu')(de_5)
# 6 decoder C512 (decodes en_3)
de_6 = UpSampling3D(size=(2, 2, 2))(de_4)
de_6 = Conv3D(filters=filter_sizes[5],
kernel_size=(4, 4, 4),
padding='same')(de_6)
de_6 = BatchNormalization(name='gen_de_bn_6', axis=bn_axis)(de_6)
de_6 = Dropout(rate=0.5)(de_6)
# de_6 = merge([de_6, en_2], mode=merge_mode, concat_axis=1)
de_6 = concatenate([de_6, en_2], axis=1)
de_6 = Activation('relu')(de_6)
# 7 decoder CD256 (decodes en_2)
de_7 = UpSampling3D(size=(2, 2, 2))(de_6)
de_7 = Conv3D(filters=filter_sizes[6],
kernel_size=(4, 4, 4),
padding='same')(de_7)
de_7 = BatchNormalization(name='gen_de_bn_7', axis=bn_axis)(de_7)
de_7 = Dropout(rate=0.5)(de_7)
# de_7 = merge([de_7, en_1], mode=merge_mode, concat_axis=1)
de_7 = concatenate([de_7, en_1], axis=1)
de_7 = Activation('relu')(de_7)
# After the last layer in the decoder, a convolution is applied
# to map to the number of output channels (3 in general,
# except in colorization, where it is 2), followed by a Tanh
# function.
de_8 = UpSampling3D(size=(2, 2, 2))(de_7)
de_8 = Conv3D(filters=num_output_channels,
kernel_size=(4, 4, 4),
padding='same')(de_8)
de_8 = Activation('tanh')(de_8)
unet_generator = Model(inputs=[input_layer],
outputs=[de_8],
name='unet_generator')
return unet_generator
def encoder_model():
inputs = Input(shape=(int(VIDEO_LENGTH / 2), 128, 208, 3))
# 10x128x128
conv_1 = Conv3D(filters=128,
strides=(1, 4, 4),
dilation_rate=(1, 1, 1),
kernel_size=(3, 11, 11),
padding='same')(inputs)
x = TimeDistributed(BatchNormalization())(conv_1)
x = TimeDistributed(LeakyReLU(alpha=0.2))(x)
out_1 = TimeDistributed(Dropout(0.5))(x)
conv_2a = Conv3D(filters=64,
strides=(1, 1, 1),
dilation_rate=(2, 1, 1),
kernel_size=(2, 5, 5),
padding='same')(out_1)
x = TimeDistributed(BatchNormalization())(conv_2a)
x = TimeDistributed(LeakyReLU(alpha=0.2))(x)
out_2a = TimeDistributed(Dropout(0.5))(x)
conv_2b = Conv3D(filters=64,
strides=(1, 1, 1),
dilation_rate=(2, 1, 1),
kernel_size=(2, 5, 5),
padding='same')(out_2a)
x = TimeDistributed(BatchNormalization())(conv_2b)
x = TimeDistributed(LeakyReLU(alpha=0.2))(x)
out_2b = TimeDistributed(Dropout(0.5))(x)
conv_2c = TimeDistributed(
Conv2D(filters=64, kernel_size=(1, 1), strides=(1, 1),
padding='same'))(out_1)
x = TimeDistributed(BatchNormalization())(conv_2c)
out_1_less = TimeDistributed(LeakyReLU(alpha=0.2))(x)
res_1 = add([out_1_less, out_2b])
# res_1 = LeakyReLU(alpha=0.2)(res_1)
conv_3 = Conv3D(filters=64,
strides=(1, 2, 2),
dilation_rate=(1, 1, 1),
kernel_size=(3, 5, 5),
padding='same')(res_1)
x = TimeDistributed(BatchNormalization())(conv_3)
x = TimeDistributed(LeakyReLU(alpha=0.2))(x)
out_3 = TimeDistributed(Dropout(0.5))(x)
# 10x16x16
conv_4a = Conv3D(filters=64,
strides=(1, 1, 1),
dilation_rate=(2, 1, 1),
kernel_size=(2, 3, 3),
padding='same')(out_3)
x = TimeDistributed(BatchNormalization())(conv_4a)
x = TimeDistributed(LeakyReLU(alpha=0.2))(x)
out_4a = TimeDistributed(Dropout(0.5))(x)
conv_4b = Conv3D(filters=64,
strides=(1, 1, 1),
dilation_rate=(2, 1, 1),
kernel_size=(2, 3, 3),
padding='same')(out_4a)
x = TimeDistributed(BatchNormalization())(conv_4b)
x = TimeDistributed(LeakyReLU(alpha=0.2))(x)
out_4b = TimeDistributed(Dropout(0.5))(x)
z = add([out_3, out_4b])
# res_1 = LeakyReLU(alpha=0.2)(res_1)
model = Model(inputs=inputs, outputs=[z, res_1])
return model
uploaded = files.upload()
modelnet_file = 'modelnet10.npz'
data = np.load(modelnet_file)
"""#Split Train and Test Data"""
X, Y = shuffle(data['X_train'], data['y_train'])
X_test, Y_test = shuffle(data['X_test'], data['y_test'])
"""# One-hot encode training targets"""
Y = keras.utils.to_categorical(Y, num_classes=10)
"""# Build the network"""
model = Sequential()
model.add(Reshape((30, 30, 30, 1), input_shape=(30, 30, 30))) # 1 in-channel
model.add(Conv3D(16, 6, strides=2, activation='relu'))
model.add(Conv3D(64, 5, strides=2, activation='relu'))
model.add(Conv3D(64, 5, strides=2, activation='relu'))
model.add(Flatten())
model.add(Dense(10, activation='softmax'))
"""# Train and Show Test Accuracy"""
model.compile(loss='categorical_crossentropy',
optimizer=keras.optimizers.Adam(lr=1e-4))
model.fit(X,
Y,
batch_size=256,
epochs=30,
verbose=2,
validation_split=0.2,
shuffle=True)
def fCreateModel(patchSize,
learningRate=1e-3,
optimizer='SGD',
dr_rate=0.0,
input_dr_rate=0.0,
max_norm=5,
iPReLU=0,
l2_reg=1e-6):
# change to functional API
input_t = Input(shape=(1, int(patchSize[0]), int(patchSize[1]),
int(patchSize[2])))
seq_t = Dropout(dr_rate)(input_t)
seq_t = Conv3D(
32, # numChans
kernel_size=(14, 14, 5),
kernel_initializer='he_normal',
weights=None,
padding='valid',
strides=(1, 1, 1),
kernel_regularizer=l2(l2_reg),
input_shape=(1, int(patchSize[0]), int(patchSize[1]),
int(patchSize[2])))(seq_t)
seq_t = fGetActivation(seq_t, iPReLU=iPReLU)
seq_t = Dropout(dr_rate)(seq_t)
seq_t = Conv3D(64,
kernel_size=(7, 7, 3),
kernel_initializer='he_normal',
weights=None,
padding='valid',
strides=(1, 1, 1),
kernel_regularizer=l2(l2_reg))(seq_t)
seq_t = fGetActivation(seq_t, iPReLU=iPReLU)
seq_t = Dropout(dr_rate)(seq_t)
seq_t = Conv3D(128,
kernel_size=(3, 3, 2),
kernel_initializer='he_normal',
weights=None,
padding='valid',
strides=(1, 1, 1),
kernel_regularizer=l2(l2_reg))(seq_t)
seq_t = fGetActivation(seq_t, iPReLU=iPReLU)
seq_t = Flatten()(seq_t)
seq_t = Dropout(dr_rate)(seq_t)
seq_t = Dense(units=2,
kernel_initializer='normal',
kernel_regularizer=l2(l2_reg))(seq_t)
output_t = Activation('softmax')(seq_t)
opti, loss = fGetOptimizerAndLoss(
optimizer, learningRate=learningRate) # loss cat_crosent default
cnn = Model(inputs=[input_t], outputs=[output_t])
cnn.compile(loss=loss, optimizer=opti, metrics=['accuracy'])
sArchiSpecs = '_l2{}'.format(l2_reg)
return cnn
def __init__(self):
self.logger.info("Assembling Model")
self._input = Input(shape=(1, 3, 9600, 3600))
"""keras.layers.Input: keras style input layer"""
conv1 = Conv3D(32, (1, 2, 2),
strides=(1, 2, 2),
activation='relu',
padding='same',
data_format='channels_first',
name='block1_conv1')(self._input)
"""keras.layers.convolutional.Conv3D: First block convolution"""
pool1 = MaxPooling3D((1, 2, 2),
strides=(1, 2, 2),
data_format='channels_first',
name='block1_pool')(conv1)
"""keras.layers.convolutional.MaxPooling3D: First block pooling"""
# Block 2
conv2 = Conv3D(64, (1, 2, 2),
strides=(1, 2, 2),
activation='relu',
padding='same',
data_format='channels_first',
name='block2_conv1')(pool1)
"""keras.layers.convolutional.Conv3D: Second block convolution"""
pool2 = MaxPooling3D((1, 2, 2),
strides=(1, 2, 2),
data_format='channels_first',
name='block2_pool')(conv2)
"""keras.layers.convolutional.MaxPooling3D: Second block pooling"""
# Block 3
conv3 = Conv3D(128, (3, 2, 2),
strides=(3, 2, 2),
activation='relu',
padding='same',
data_format='channels_first',
name='block3_conv1')(pool2)
"""keras.layers.convolutional.Conv3D: Third block convolution"""
pool3 = MaxPooling3D((1, 2, 2),
strides=(1, 2, 2),
data_format='channels_first',
name='block3_pool')(conv3)
"""keras.layers.convolutional.Conv3D: Third block convolution"""
# Block 4
conv4 = Conv3D(256, (1, 2, 2),
strides=(1, 2, 2),
activation='relu',
padding='same',
data_format='channels_first',
name='block4_conv1')(pool3)
"""keras.layers.convolutional.Conv3D: Fourth block convolution"""
pool4 = MaxPooling3D((1, 2, 2),
strides=(1, 2, 2),
name='block4_pool',
data_format='channels_first')(conv4)
# Block 5
conv5 = Conv3D(512, (1, 2, 2),
strides=(1, 2, 2),
activation='relu',
padding='same',
data_format='channels_first')(pool4)
"""keras.layers.convolutional.Conv3D: Fifth block convolution"""
pool5 = MaxPooling3D((1, 2, 2),
strides=(1, 2, 2),
name='block5_pool',
data_format='channels_first')(conv5)
super(aerophilatelic_pichi, self).__init__(self._input, up7)
def discriminator(
trainable=True,
params={
'input_dim': 64,
'strides': (2, 2, 2),
'kernel_size': (4, 4, 4),
'leak_value': 0.2
}):
"""
Returns a Discriminator Model
Args:
trainable (boolean): Train the model or not
params (dict): Dictionary with model parameters
Returns:
model (keras.Model): Discriminator model
"""
input_dim = params['input_dim']
strides = params['strides']
kernel_size = params['kernel_size']
leak_value = params['leak_value']
inputs = Input(shape=(input_dim, input_dim, input_dim, 1))
discriminator_l1 = Conv3D(filters=64,
kernel_size=kernel_size,
strides=strides,
kernel_initializer='glorot_normal',
bias_initializer='zeros',
padding='same')(inputs)
discriminator_l1 = BatchNormalization()(discriminator_l1,
training=trainable)
discriminator_l1 = LeakyReLU(leak_value)(discriminator_l1)
discriminator_l2 = Conv3D(filters=128,
kernel_size=kernel_size,
strides=strides,
kernel_initializer='glorot_normal',
bias_initializer='zeros',
padding='same')(discriminator_l1)
discriminator_l2 = BatchNormalization()(discriminator_l2,
training=trainable)
discriminator_l2 = LeakyReLU(leak_value)(discriminator_l2)
discriminator_l3 = Conv3D(filters=256,
kernel_size=kernel_size,
strides=strides,
kernel_initializer='glorot_normal',
bias_initializer='zeros',
padding='same')(discriminator_l2)
discriminator_l3 = BatchNormalization()(discriminator_l3,
training=trainable)
discriminator_l3 = LeakyReLU(leak_value)(discriminator_l3)
discriminator_l4 = Conv3D(filters=512,
kernel_size=kernel_size,
strides=strides,
kernel_initializer='glorot_normal',
bias_initializer='zeros',
padding='same')(discriminator_l3)
discriminator_l4 = BatchNormalization()(discriminator_l4,
training=trainable)
discriminator_l4 = LeakyReLU(leak_value)(discriminator_l4)
discriminator_l5 = Conv3D(filters=1,
kernel_size=kernel_size,
strides=(1, 1, 1),
kernel_initializer='glorot_normal',
bias_initializer='zeros',
padding='valid')(discriminator_l4)
discriminator_l5 = BatchNormalization()(discriminator_l5,
training=trainable)
discriminator_l5 = Activation(activation='sigmoid')(discriminator_l5)
model = Model(inputs=inputs, outputs=discriminator_l5)
print("Discriminator")
model.summary()
return model
while True:
start = np.random.randint(TOTAL_SAMPLES - batch_size)
end = start + batch_size
bdf = df.iloc[start:end]
x_train = get_x_batch(bdf)
y_train = Y_values[start:end, :]
yield x_train, y_train
model = Sequential()
# 1st layer group
model.add(
Conv3D(64, (3, 3, 3),
activation="relu",
padding="same",
name='conv1',
strides=(1, 1, 1),
input_shape=(frames_in_each_sample, frame_shape[1], frame_shape[0],
channels)))
model.add(
MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
padding="valid",
name='pool1'))
# 2nd layer group
model.add(
Conv3D(128, (3, 3, 3),
activation="relu",
padding="same",
name='conv2',
strides=(1, 1, 1)))
def keras_model(layers=3,
filters=32,
kernel_size=(3, 3),
activation='relu',
recurrent_activation='tanh',
dropout=0.0,
recurrent_dropout=0.0):
input_shape = (None, 50, 150, 4)
seq = Sequential()
if layers >= 1:
seq.add(
ConvLSTM2D(filters=filters,
kernel_size=kernel_size,
activation=activation,
input_shape=input_shape,
recurrent_activation=recurrent_activation,
padding='same',
dropout=dropout,
recurrent_dropout=recurrent_dropout,
return_sequences=True))
if layers >= 2:
seq.add(
ConvLSTM2D(filters=filters,
kernel_size=kernel_size,
activation=activation,
recurrent_activation=recurrent_activation,
padding='same',
dropout=dropout,
recurrent_dropout=recurrent_dropout,
return_sequences=True))
if layers >= 3:
seq.add(
ConvLSTM2D(filters=filters,
kernel_size=kernel_size,
activation=activation,
recurrent_activation=recurrent_activation,
padding='same',
dropout=dropout,
recurrent_dropout=recurrent_dropout,
return_sequences=True))
if layers >= 4:
seq.add(
ConvLSTM2D(filters=filters,
kernel_size=kernel_size,
activation=activation,
recurrent_activation=recurrent_activation,
padding='same',
dropout=dropout,
recurrent_dropout=recurrent_dropout,
return_sequences=True))
if layers >= 5:
seq.add(
ConvLSTM2D(filters=filters,
kernel_size=kernel_size,
activation=activation,
recurrent_activation=recurrent_activation,
padding='same',
dropout=dropout,
recurrent_dropout=recurrent_dropout,
return_sequences=True))
if layers >= 6:
seq.add(
ConvLSTM2D(filters=filters,
kernel_size=kernel_size,
activation=activation,
recurrent_activation=recurrent_activation,
padding='same',
dropout=dropout,
recurrent_dropout=recurrent_dropout,
return_sequences=True))
if layers >= 7:
seq.add(
ConvLSTM2D(filters=filters,
kernel_size=kernel_size,
activation=activation,
recurrent_activation=recurrent_activation,
padding='same',
dropout=dropout,
recurrent_dropout=recurrent_dropout,
return_sequences=True))
if layers >= 8:
seq.add(
ConvLSTM2D(filters=filters,
kernel_size=kernel_size,
activation=activation,
recurrent_activation=recurrent_activation,
padding='same',
dropout=dropout,
recurrent_dropout=recurrent_dropout,
return_sequences=True))
if layers >= 9:
seq.add(
ConvLSTM2D(filters=filters,
kernel_size=kernel_size,
activation=activation,
recurrent_activation=recurrent_activation,
padding='same',
dropout=dropout,
recurrent_dropout=recurrent_dropout,
return_sequences=True))
seq.add(
Conv3D(filters=4,
kernel_size=(3, 3, 3),
activation='linear',
padding='same',
data_format='channels_last'))
return seq
def get_model(summary=False, backend='tf'):
""" Return the Keras model of the network
"""
model = Sequential()
if backend == 'tf':
input_shape = (16, 112, 112, 3) # l, h, w, c
else:
input_shape = (3, 16, 112, 112) # c, l, h, w
model.add(
Conv3D(64, (3, 3, 3),
activation='relu',
padding='same',
name='conv1',
input_shape=input_shape))
model.add(
MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
padding='valid',
name='pool1'))
# 2nd layer group
model.add(
Conv3D(128, (3, 3, 3), activation='relu', padding='same',
name='conv2'))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
padding='valid',
name='pool2'))
# 3rd layer group
model.add(
Conv3D(256, (3, 3, 3),
activation='relu',
padding='same',
name='conv3a'))
model.add(
Conv3D(256, (3, 3, 3),
activation='relu',
padding='same',
name='conv3b'))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
padding='valid',
name='pool3'))
# 4th layer group
model.add(
Conv3D(512, (3, 3, 3),
activation='relu',
padding='same',
name='conv4a'))
model.add(
Conv3D(512, (3, 3, 3),
activation='relu',
padding='same',
name='conv4b'))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
padding='valid',
name='pool4'))
# 5th layer group
model.add(
Conv3D(512, (3, 3, 3),
activation='relu',
padding='same',
name='conv5a'))
model.add(
Conv3D(512, (3, 3, 3),
activation='relu',
padding='same',
name='conv5b'))
model.add(ZeroPadding3D(padding=(0, 1, 1), name='zeropad5'))
model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
padding='valid',
name='pool5'))
model.add(Flatten())
# FC layers group
model.add(Dense(4096, activation='relu', name='fc6'))
model.add(Dropout(.5))
model.add(Dense(4096, activation='relu', name='fc7'))
model.add(Dropout(.5))
model.add(Dense(487, activation='softmax', name='fc8'))
if summary:
print(model.summary())
return model
def build(self):
if K.image_data_format() == 'channels_first':
input_shape = (self.frames_n, self.img_c, self.img_w, self.img_h)
else:
input_shape = (self.frames_n, self.img_w, self.img_h, self.img_c)
self.input_data = Input(name='the_input',
shape=input_shape,
dtype='float32')
self.zero1 = ZeroPadding3D(padding=(1, 2, 2),
name='zero1')(self.input_data)
self.conv1 = Conv3D(32, (3, 5, 5),
strides=(1, 2, 2),
activation='relu',
kernel_initializer='he_normal',
name='conv1')(self.zero1)
self.maxp1 = MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
name='max1')(self.conv1)
self.drop1 = Dropout(0.5)(self.maxp1)
self.zero2 = ZeroPadding3D(padding=(1, 2, 2), name='zero2')(self.drop1)
self.conv2 = Conv3D(64, (3, 5, 5),
strides=(1, 1, 1),
activation='relu',
kernel_initializer='he_normal',
name='conv2')(self.zero2)
self.maxp2 = MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
name='max2')(self.conv2)
self.drop2 = Dropout(0.5)(self.maxp2)
self.zero3 = ZeroPadding3D(padding=(1, 1, 1), name='zero3')(self.drop2)
self.conv3 = Conv3D(96, (3, 3, 3),
strides=(1, 1, 1),
activation='relu',
kernel_initializer='he_normal',
name='conv3')(self.zero3)
self.maxp3 = MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
name='max3')(self.conv3)
self.drop3 = Dropout(0.5)(self.maxp3)
self.resh1 = TimeDistributed(Flatten())(self.drop3)
self.gru_1 = Bidirectional(GRU(256,
return_sequences=True,
kernel_initializer='Orthogonal',
name='gru1'),
merge_mode='concat')(self.resh1)
self.gru_2 = Bidirectional(GRU(256,
return_sequences=True,
kernel_initializer='Orthogonal',
name='gru2'),
merge_mode='concat')(self.gru_1)
# transforms RNN output to character activations:
self.dense1 = Dense(self.output_size,
kernel_initializer='he_normal',
name='dense1')(self.gru_2)
self.y_pred = Activation('softmax', name='softmax')(self.dense1)
self.labels = Input(name='the_labels',
shape=[self.absolute_max_string_len],
dtype='float32')
self.input_length = Input(name='input_length',
shape=[1],
dtype='int64')
self.label_length = Input(name='label_length',
shape=[1],
dtype='int64')
self.loss_out = CTC(
'ctc',
[self.y_pred, self.labels, self.input_length, self.label_length])
self.model = Model(inputs=[
self.input_data, self.labels, self.input_length, self.label_length
],
outputs=self.loss_out)
def build_generator(self):
"""U-Net Generator"""
def conv3d(layer_input,
filters,
kernel_size=(4, 4, 2),
strides=(2, 2, 2),
bn=True):
"""Layers used during downsampling"""
init = RandomNormal(stddev=0.02)
d = Conv3D(filters,
kernel_size=kernel_size,
strides=strides,
padding='same',
kernel_initializer=init)(layer_input)
d = LeakyReLU(alpha=0.2)(d)
if bn:
#d = BatchNormalization(momentum=0.8)(d)
d = InstanceNormalization()(d)
return d
def deconv3d(layer_input,
skip_input,
filters,
kernel_size=(4, 4, 2),
strides=(2, 2, 2),
dropout_rate=0,
bn=True):
"""Layers used during upsampling"""
if self.resizeconv:
u = My3dResize(strides)(layer_input)
else:
u = UpSampling3D(size=strides)(layer_input)
init = RandomNormal(stddev=0.02)
u = Conv3D(filters,
kernel_size=kernel_size,
strides=1,
padding='same',
kernel_initializer=init,
activation='relu')(u)
if dropout_rate:
u = Dropout(dropout_rate)(u)
if bn:
#u = BatchNormalization(momentum=0.8)(u)
u = InstanceNormalization()(u)
u = Concatenate()([u, skip_input])
u = Activation('relu')(u)
return u
# Image input
d00 = Input(shape=self.img_shape)
if self.coordconv:
d0 = CoordinateChannel3D()(d00)
else:
d0 = d00
n_layers = 7
encoders = []
decoders = []
# Downsampling
for i in range(n_layers):
z = 1
if i < self.depth.bit_length() - 1: z = 2
if i == 0:
encoders.append(
conv3d(d0,
self.gf,
kernel_size=(4, 4, z),
strides=(2, 2, z),
bn=False))
else:
encoders.append(
conv3d(encoders[-1],
self.gf * (2**min(i, 3)),
kernel_size=(4, 4, z),
strides=(2, 2, z)))
# Upsampling
for i in range(n_layers - 1):
z = 1
if i + self.depth.bit_length() > n_layers: z = 2
if i == 0:
decoders.append(
deconv3d(encoders[-(i + 1)],
encoders[-(i + 2)],
self.gf * (2**min(n_layers - 2 - i, 3)),
kernel_size=(4, 4, z),
strides=(2, 2, z)))
else:
decoders.append(
deconv3d(decoders[-1],
encoders[-(i + 2)],
self.gf * (2**min(n_layers - 2 - i, 3)),
kernel_size=(4, 4, z),
strides=(2, 2, z),
dropout_rate=self.dropout))
if self.resizeconv:
u7 = My3dResize((2, 2, 2))(decoders[-1])
else:
u7 = UpSampling3D(size=2)(decoders[-1])
init = RandomNormal(stddev=0.02)
output_img = Conv3D(self.channels,
kernel_size=(4, 4, 2),
strides=1,
padding='same',
kernel_initializer=init,
activation='tanh')(u7)
return Model(d00, output_img)
seq.add(
ConvLSTM2D(filters=40,
kernel_size=(3, 3),
padding='same',
return_sequences=True))
seq.add(BatchNormalization())
seq.add(
ConvLSTM2D(filters=40,
kernel_size=(3, 3),
padding='same',
return_sequences=True))
seq.add(BatchNormalization())
seq.add(
Conv3D(
filters=1, # binary output
kernel_size=(3, 3, 3),
activation='sigmoid',
padding='same',
data_format='channels_last')
) # channel is the last dim of the output tensor
seq.compile(loss='binary_crossentropy', optimizer='adadelta')
# create movies with bigger size (80x80) and then select a 40x40 window
def generate_movies(n_samples=1200, n_frames=15):
row = 80
col = 80
noisy_movies = np.zeros((n_samples, n_frames, row, col, 1), dtype=np.float)
shifted_movies = np.zeros((n_samples, n_frames, row, col, 1),
dtype=np.float)
for i in range(n_samples):
import tensorflow as tf config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.9 tf.keras.backend.set_session(tf.Session(config=config)) from keras.layers.convolutional import Conv3D, MaxPooling3D from keras.layers.core import Dense, Dropout, Flatten from keras.models import Sequential from keras.layers import Activation from tensorflow import keras from tensorflow.keras.models import load_model model = Sequential() model.add(Conv3D(16, (6, 6, 6), strides = 1, input_shape=(20, 100, 50, 1), activation='relu', padding='valid')) model.add(MaxPooling3D(pool_size=(2, 2, 2))) model.add(Dropout(0.5)) model.add(Conv3D(8, (3, 3, 3), strides = 1, activation='relu', padding='valid')) model.add(MaxPooling3D(pool_size=(2, 2, 2))) model.add(Dropout(0.5)) model.add(Flatten()) model.add(Dense(64, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(32, activation='relu')) model.add(Dropout(0.5))
def get_network(options):
"""
CNN model for MS lesion segmentation. Based on the model proposed on:
Valverde, S. et al (2017). Improving automated multiple sclerosis lesion
segmentation with a cascaded 3D convolutional neural network approach.
NeuroImage, 155, 159-168. https://doi.org/10.1016/j.neuroimage.2017.04.034
However, two additional fully-connected layers are added to increase
the effective transfer learning
"""
# model options
channels = len(options['modalities'])
net_input1 = Input(name='in1', shape=(channels,) + options['patch_size'])
net_input2 = Input(name='in2', shape=(channels,) + options['patch_size'])
net_input3 = Input(name='in3', shape=(channels,) + options['patch_size'])
net_input4 = Input(name='in4', shape=(channels,) + options['patch_size'])
net_input5 = Input(name='in5', shape=(channels,) + options['patch_size'])
merged = keras.layers.Concatenate(axis=1)([net_input1,net_input2,net_input3,net_input4,net_input5])
# net_input = [net_input1,net_input2,net_input3,net_input4,net_input5]
layer = Conv3D(filters=32, kernel_size=(3, 3, 3),
name='conv1_1',
activation=None,
padding="same")(merged)
layer = BN(name='bn_1_1', axis=1)(layer)
layer = prelu(name='prelu_conv1_1')(layer)
layer = Conv3D(filters=32,
kernel_size=(3, 3, 3),
name='conv1_2',
activation=None,
padding="same")(layer)
layer = BN(name='bn_1_2', axis=1)(layer)
layer = prelu(name='prelu_conv1_2')(layer)
layer = MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2))(layer)
layer = Conv3D(filters=64,
kernel_size=(3, 3, 3),
name='conv2_1',
activation=None,
padding="same")(layer)
layer = BN(name='bn_2_1', axis=1)(layer)
layer = prelu(name='prelu_conv2_1')(layer)
layer = Conv3D(filters=64,
kernel_size=(3, 3, 3),
name='conv2_2',
activation=None,
padding="same")(layer)
layer = BN(name='bn_2_2', axis=1)(layer)
layer = prelu(name='prelu_conv2_2')(layer)
layer = MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2))(layer)
layer = Flatten()(layer)
layer = Dropout(name='dr_d1', rate=0.5)(layer)
layer = Dense(units=256, activation=None, name='d1')(layer)
layer = prelu(name='prelu_d1')(layer)
layer = Dropout(name='dr_d2', rate=0.5)(layer)
layer = Dense(units=128, activation=None, name='d2')(layer)
layer = prelu(name='prelu_d2')(layer)
layer = Dropout(name='dr_d3', rate=0.5)(layer)
layer = Dense(units=64, activation=None, name='d3')(layer)
layer = prelu(name='prelu_d3')(layer)
# net_output = Dense(units=2, name='out', activation='softmax')(layer)
output1 = Dense(units=2, name='output_label1', activation='softmax')(layer)
output2 = Dense(units=2, name='output_label2', activation='softmax')(layer)
output3 = Dense(units=2, name='output_label3', activation='softmax')(layer)
output4 = Dense(units=2, name='output_label4', activation='softmax')(layer)
output5 = Dense(units=2, name='output_label5', activation='softmax')(layer)
# model = Model(inp,[output1,output2,output3,output4,output5])
# output1 = Dense(1, activation = 'sigmoid')(x)
# output2 = Dense(1, activation = 'sigmoid')(x)
# output3 = Dense(1, activation = 'sigmoid')(x)
# output4 = Dense(1, activation = 'sigmoid')(x)
# output5 = Dense(1, activation = 'sigmoid')(x)
# net_output = [output1,output2,output3,output4,output5,output6,output7,output8,output9,output10,output11,output12]
model = Model(inputs=[net_input1,net_input2,net_input3,net_input4,net_input5], outputs=[output1,output2,output3,output4,output5])
# model = Model(inputs=[net_input], outputs=net_output)
return model
def get_int_model(model, layer, backend='tf'):
if backend == 'tf':
input_shape = (16, 112, 112, 3) # l, h, w, c
else:
input_shape = (3, 16, 112, 112) # c, l, h, w
int_model = Sequential()
int_model.add(
Conv3D(64,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv1',
input_shape=input_shape,
weights=model.layers[0].get_weights()))
if layer == 'conv1':
return int_model
int_model.add(
MaxPooling3D(pool_size=(1, 2, 2),
strides=(1, 2, 2),
border_mode='valid',
name='pool1'))
if layer == 'pool1':
return int_model
# 2nd layer group
int_model.add(
Conv3D(128,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv2',
weights=model.layers[2].get_weights()))
if layer == 'conv2':
return int_model
int_model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool2'))
if layer == 'pool2':
return int_model
# 3rd layer group
int_model.add(
Conv3D(256,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv3a',
weights=model.layers[4].get_weights()))
if layer == 'conv3a':
return int_model
int_model.add(
Conv3D(256,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv3b',
weights=model.layers[5].get_weights()))
if layer == 'conv3b':
return int_model
int_model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool3'))
if layer == 'pool3':
return int_model
# 4th layer group
int_model.add(
Conv3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv4a',
weights=model.layers[7].get_weights()))
if layer == 'conv4a':
return int_model
int_model.add(
Conv3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv4b',
weights=model.layers[8].get_weights()))
if layer == 'conv4b':
return int_model
int_model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool4'))
if layer == 'pool4':
return int_model
# 5th layer group
int_model.add(
Conv3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv5a',
weights=model.layers[10].get_weights()))
if layer == 'conv5a':
return int_model
int_model.add(
Conv3D(512,
3,
3,
3,
activation='relu',
border_mode='same',
name='conv5b',
weights=model.layers[11].get_weights()))
if layer == 'conv5b':
return int_model
int_model.add(ZeroPadding3D(padding=(0, 1, 1), name='zeropad'))
int_model.add(
MaxPooling3D(pool_size=(2, 2, 2),
strides=(2, 2, 2),
border_mode='valid',
name='pool5'))
if layer == 'pool5':
return int_model
int_model.add(Flatten())
# FC layers group
int_model.add(
Dense(4096,
activation='relu',
name='fc6',
weights=model.layers[15].get_weights()))
if layer == 'fc6':
return int_model
int_model.add(Dropout(.5))
int_model.add(
Dense(4096,
activation='relu',
name='fc7',
weights=model.layers[17].get_weights()))
if layer == 'fc7':
return int_model
int_model.add(Dropout(.5))
int_model.add(
Dense(487,
activation='softmax',
name='fc8',
weights=model.layers[19].get_weights()))
if layer == 'fc8':
return int_model
return None
def fCreateModel(patchSize,
learningRate=1e-3,
optimizer='SGD',
dr_rate=0.0,
input_dr_rate=0.0,
max_norm=5,
iPReLU=0,
l2_reg=1e-6,
architecture='Layers4'):
# change to functional API
if architecture == 'Layers4':
cnn = Sequential()
cnn.add(
Conv3D(
32, # numChans
kernel_size=(6, 6, 6),
kernel_initializer='he_normal',
weights=None,
padding='valid',
strides=(1, 1, 1),
kernel_regularizer=l2(1e-6),
# tricked input shape, patchsize is stored as float...
input_shape=(1, int(patchSize[0, 0]), int(patchSize[0, 1]),
int(patchSize[0, 2]))))
cnn.add(Activation('relu'))
cnn.add(
Conv3D(64,
kernel_size=(5, 5, 5),
kernel_initializer='he_normal',
weights=None,
padding='valid',
strides=(1, 1, 1),
kernel_regularizer=l2(1e-6)))
cnn.add(Activation('relu'))
cnn.add(
Conv3D(128,
kernel_size=(4, 4, 4),
kernel_initializer='he_normal',
weights=None,
padding='valid',
strides=(1, 1, 1),
kernel_regularizer=l2(1e-6)))
cnn.add(Activation('relu'))
cnn.add(
Conv3D(128,
kernel_size=(3, 3, 3),
kernel_initializer='he_normal',
weights=None,
padding='valid',
strides=(1, 1, 1),
kernel_regularizer=l2(1e-6)))
cnn.add(Activation('relu'))
cnn.add(Flatten())
cnn.add(
Dense(units=2,
kernel_initializer='normal',
kernel_regularizer='l2'))
cnn.add(Activation('softmax'))
opti, loss = h.fGetOptimizerAndLoss(
optimizer, learningRate=learningRate) # loss cat_crosent default
cnn.compile(loss=loss, optimizer=opti, metrics=['accuracy'])
elif architecture == 'Layers4_dropout':
cnn = Sequential()
cnn.add(
Dropout(
input_shape=(1, int(patchSize[0, 0]), int(patchSize[0, 1]),
int(patchSize[0, 2])),
rate=input_dr_rate,
))
cnn.add(
Conv3D(
32, # number of channels should be #wo_dr *(1/p) =>leave them how they are
kernel_size=(6, 6, 6),
kernel_initializer='he_normal',
weights=None,
padding='valid',
strides=(1, 1, 1),
kernel_regularizer=l2(l2),
))
cnn.add(Activation('relu'))
cnn.add(Dropout(rate=dr_rate))
cnn.add(
Conv3D(64,
kernel_size=(5, 5, 5),
kernel_initializer='he_normal',
weights=None,
padding='valid',
strides=(1, 1, 1),
kernel_regularizer=l2(l2)))
cnn.add(Activation('relu'))
cnn.add(Dropout(rate=dr_rate))
cnn.add(
Conv3D(128,
kernel_size=(4, 4, 4),
kernel_initializer='he_normal',
weights=None,
padding='valid',
strides=(1, 1, 1),
kernel_regularizer=l2(l2)))
cnn.add(Activation('relu'))
cnn.add(Dropout(rate=dr_rate))
cnn.add(
Conv3D(128,
kernel_size=(3, 3, 3),
kernel_initializer='he_normal',
weights=None,
padding='valid',
strides=(1, 1, 1),
kernel_regularizer=l2(l2)))
cnn.add(Activation('relu'))
cnn.add(Flatten())
cnn.add(Dropout(rate=dr_rate))
cnn.add(
Dense(units=2,
kernel_initializer='normal',
kernel_regularizer='l2'))
cnn.add(Activation('softmax'))
opti, loss = h.fGetOptimizerAndLoss(
optimizer, learningRate=learningRate) # loss categ_crosent default
cnn.compile(loss=loss, optimizer=opti, metrics=['accuracy'])
elif architecture == 'Layers4_dropout_maxnorm':
cnn = Sequential()
cnn.add(
Dropout(rate=input_dr_rate,
input_shape=(1, int(patchSize[0, 0]), int(patchSize[0, 1]),
int(patchSize[0, 2]))))
cnn.add(
Conv3D(
64, # numChans
kernel_size=(6, 6, 6),
kernel_initializer='he_normal',
weights=None,
padding='valid',
strides=(1, 1, 1),
kernel_constraint=maxnorm(
max_value=max_norm), #see dropout reference
kernel_regularizer=l2(1e-6),
))
cnn.add(Activation('relu'))
cnn.add(Dropout(rate=dr_rate))
cnn.add(
Conv3D(
128,
kernel_size=(5, 5, 5),
kernel_initializer='he_normal',
kernel_constraint=maxnorm(
max_value=max_norm), # see dropout reference
weights=None,
padding='valid',
strides=(1, 1, 1),
kernel_regularizer=l2(1e-6)))
cnn.add(Activation('relu'))
cnn.add(Dropout(rate=dr_rate))
cnn.add(
Conv3D(
256,
kernel_size=(4, 4, 4),
kernel_initializer='he_normal',
kernel_constraint=maxnorm(
max_value=max_norm), # see dropout reference
weights=None,
padding='valid',
strides=(1, 1, 1),
kernel_regularizer=l2(1e-6)))
cnn.add(Activation('relu'))
cnn.add(Dropout(rate=dr_rate))
cnn.add(
Conv3D(
256,
kernel_size=(3, 3, 3),
kernel_initializer='he_normal',
kernel_constraint=maxnorm(
max_value=max_norm), # see dropout reference
weights=None,
padding='valid',
strides=(1, 1, 1),
kernel_regularizer=l2(1e-6)))
cnn.add(Activation('relu'))
cnn.add(Flatten())
cnn.add(Dropout(rate=dr_rate))
cnn.add(
Dense(
units=2,
kernel_initializer='normal',
kernel_constraint=maxnorm(
max_value=max_norm), # see dropout reference
kernel_regularizer='l2'))
cnn.add(Activation('softmax'))
opti, loss = h.fGetOptimizerAndLoss(
optimizer, learningRate=learningRate) # loss cat_crosent default
cnn.compile(loss=loss, optimizer=opti, metrics=['accuracy'])
return cnn, architecture
seq.add(BatchNormalization())
seq.add(ConvLSTM2D(filters=40, kernel_size=(3, 3),
padding='same', return_sequences=True))
seq.add(BatchNormalization())
seq.add(ConvLSTM2D(filters=40, kernel_size=(3, 3),
padding='same', return_sequences=True))
seq.add(BatchNormalization())
seq.add(ConvLSTM2D(filters=40, kernel_size=(3, 3),
padding='same', return_sequences=True))
seq.add(BatchNormalization())
seq.add(Conv3D(filters=1, kernel_size=(3, 3, 3),
activation='sigmoid',
padding='same', data_format='channels_last'))
seq.compile(loss='binary_crossentropy', optimizer='adadelta')
# Artificial data generation:
# Generate movies with 3 to 7 moving squares inside.
# The squares are of shape 1x1 or 2x2 pixels,
# which move linearly over time.
# For convenience we first create movies with bigger width and height (80x80)
# and at the end we select a 40x40 window.
def generate_movies(n_samples=1200, n_frames=15):
row = 80
col = 80
noisy_movies = np.zeros((n_samples, n_frames, row, col, 1), dtype=np.float)
def assemble(self, generator):
self.logger.info("Assembling Model")
self._input = Input(shape=generator.output)
self.logger.info(self._input)
self.logger.info(self._input.shape)
layer = Conv3D(32, (1, 5, 5),
strides=(1, 4, 4),
activation='relu',
padding='same',
data_format='channels_first',
name='block1_conv1')(self._input)
self.logger.info(layer.shape)
layer = MaxPooling3D((1, 5, 3),
strides=(1, 4, 2),
data_format='channels_first',
name='block1_pool')(layer)
self.logger.info(layer.shape)
layer = Conv3D(64, (1, 3, 3),
strides=(1, 2, 2),
activation='relu',
padding='same',
data_format='channels_first',
name='block2_conv1')(layer)
self.logger.info(layer.shape)
layer = MaxPooling3D((1, 3, 3),
strides=(1, 2, 2),
data_format='channels_first',
name='block2_pool')(layer)
self.logger.info(layer.shape)
layer = Conv3D(128, (3, 3, 3),
strides=(3, 2, 2),
activation='relu',
padding='same',
data_format='channels_first',
name='block3_conv1')(layer)
self.logger.info(layer.shape)
layer = MaxPooling3D((1, 3, 3),
strides=(1, 2, 2),
data_format='channels_first',
name='block3_pool')(layer)
self.logger.info(layer.shape)
layer = Conv3D(256, (1, 3, 3),
strides=(1, 2, 2),
activation='relu',
padding='same',
data_format='channels_first',
name='block4_conv1')(layer)
self.logger.info(layer.shape)
layer = MaxPooling3D((1, 3, 3),
strides=(1, 2, 2),
data_format='channels_first',
name='block4_pool')(layer)
self.logger.info(layer.shape)
layer = Conv3D(512, (1, 3, 3),
strides=(1, 2, 2),
activation='relu',
padding='same',
data_format='channels_first',
name='block5_conv1')(layer)
self.logger.info(layer.shape)
layer = MaxPooling3D((1, 3, 3),
strides=(1, 2, 2),
data_format='channels_first',
name='block5_pool')(layer)
self.logger.info(layer.shape)
layer = Dropout(0.1)(layer)
# Classification block
layer = Flatten(name='flatten')(layer)
layer = Dropout(0.01)(layer)
layer = Dense(1024, activation='relu', name='fc1')(layer)
layer = Dropout(0.01)(layer)
layer = Dense(generator.input,
activation='softmax',
name='predictions')(layer)
self.logger.info(layer.shape)
return layer
