def VGGFace_multimodal(input_shape, n_class):
"""
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:return: Keras Model used for training
"""
# RGB MODALITY BRANCH OF CNN
inputs_rgb = layers.Input(shape=input_shape)
# ########################VGG/RESNET or any other network
resnet_model_rgb = RESNET50(include_top=False,
weights='vggface',
input_tensor=None,
input_shape=None,
pooling=None,
type_name='rgb')
conv_model_rgb = resnet_model_rgb(inputs_rgb)
#
########################
inputs_depth = layers.Input(shape=input_shape)
resnet_model_depth = RESNET50(include_top=False,
weights='vggface',
input_tensor=None,
input_shape=None,
pooling=None,
type_name='depth')
# pre_trained_model = VGGFace(include_top=False, input_shape=input_shape)
# pre_trained_model.summary()
conv_model_depth = resnet_model_depth(inputs_depth)
merge_rgb_depth = layers.concatenate([conv_model_rgb, conv_model_depth],
axis=-1)
attention_features = cbam_block(merge_rgb_depth)
flat_model = layers.Flatten(name='flatten')(attention_features)
fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model)
dropout_1 = layers.Dropout(0.2)(fc6)
# flatten_concat = layers.Flatten(name='flatten_concat')(merge_rgb_depth)
# fc6 = layers.Dense(2048, activation='relu', name='fc6')(merge_rgb_depth)
fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1)
dropout_2 = layers.Dropout(0.2)(fc7)
#VECTORIZING OUTPUT
output = layers.Dense(n_class, activation='softmax',
name='output')(dropout_2)
# MODAL [INPUTS , OUTPUTS]
train_model = models.Model(inputs=[inputs_rgb, inputs_depth],
outputs=[output])
train_model.summary()
for layer in train_model.layers[:-19]:
layer.trainable = False
return train_model
def sc_block(input_map):
block_conv1 = keras.layers.Conv2D(64, (3, 3), padding='same')(input_map)
block_bn1 = keras.layers.BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001)(block_conv1)
block_Lrelu1 = keras.layers.LeakyReLU(alpha=0.05)(block_bn1)
block_concat1 = keras.layers.concatenate([input_map, block_Lrelu1], axis=-1)
block_conv2 = keras.layers.Conv2D(64, (3, 3), padding='same')(block_concat1)
block_sc = cbam_block(block_conv2)
block_concat2 = keras.layers.concatenate([input_map, block_sc], axis=-1)
return block_concat2
def VGGFace_multimodal(input_shape, n_class):
"""
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:return: Keras Model used for training
"""
# RGB MODALITY BRANCH OF CNN
######################
# conv_model_depth = vgg_model_depth(inputs_depth)
# fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(dropout_rgb)
# fc6_depth = layers.Dense(2048, activation='relu', name='fc6_depth')(dropout_depth)
# CONACTENATE the ends of RGB & DEPTH
alpha = 0.7
merge_rgb_depth = layers.concatenate([conv_model_rgb, conv_model_depth],
axis=-1)
attention_features = cbam_block(merge_rgb_depth)
#
#
# ############ for RGB
# flat_model_rgb = layers.Flatten(name='flatten_rgb')(conv_model_rgb)
# fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(flat_model_rgb)
# dropout_rgb = layers.Dropout(0.2)(fc6_rgb)
######## for Depth
flat_model = layers.Flatten(name='flatten')(attention_features)
fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model)
dropout_1 = layers.Dropout(0.2)(fc6)
# flatten_concat = layers.Flatten(name='flatten_concat')(merge_rgb_depth)
# fc6 = layers.Dense(2048, activation='relu', name='fc6')(merge_rgb_depth)
fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1)
dropout_2 = layers.Dropout(0.2)(fc7)
#VECTORIZING OUTPUT
output = layers.Dense(n_class, activation='softmax',
name='output')(dropout_2)
# MODAL [INPUTS , OUTPUTS]
train_model = models.Model(inputs=[inputs_rgb, inputs_depth],
outputs=[output])
train_model.summary()
for layer in train_model.layers[:-19]:
layer.trainable = False
return train_model
def VGGFace_multimodal(input_shape, n_class):
"""
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:return: Keras Model used for training
"""
# RGB MODALITY BRANCH OF CNN
inputs_rgb = layers.Input(shape=input_shape)
########################VGG/RESNET or any other network
vgg_model_rgb = VGGFace(include_top=False, input_shape=input_shape)
########################
conv_model_rgb = vgg_model_rgb(inputs_rgb)
## get weights
pickle_in = open("vgg16_face_conv_weights_name.pkl", "rb")
w_and_b = pickle.load(pickle_in)
# DEPTH MODALITY BRANCH OF CNN
inputs_depth = layers.Input(shape=input_shape)
#######################VGG/RESNET or any other network
# vgg_model_depth = VGGFace(include_top=False, input_shape=input_shape)
# Block 1
x = layers.Conv2D(
64, (3, 3),
activation='relu',
padding='same',
kernel_initializer=keras.initializers.Constant(w_and_b['conv1_1'][0]),
bias_initializer=keras.initializers.Constant(w_and_b['conv1_1'][1]),
name='conv1_1d')(inputs_depth)
x = layers.Conv2D(
64, (3, 3),
activation='relu',
padding='same',
kernel_initializer=keras.initializers.Constant(w_and_b['conv1_2'][0]),
bias_initializer=keras.initializers.Constant(w_and_b['conv1_2'][1]),
name='conv1_2d')(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool1d')(x)
# Block 2
x = layers.Conv2D(
128, (3, 3),
activation='relu',
padding='same',
kernel_initializer=keras.initializers.Constant(w_and_b['conv2_1'][0]),
bias_initializer=keras.initializers.Constant(w_and_b['conv2_1'][1]),
name='conv2_1d')(x)
x = layers.Conv2D(
128, (3, 3),
activation='relu',
padding='same',
kernel_initializer=keras.initializers.Constant(w_and_b['conv2_2'][0]),
bias_initializer=keras.initializers.Constant(w_and_b['conv2_1'][1]),
name='conv2_2d')(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool2d')(x)
# Block 3
x = layers.Conv2D(
256, (3, 3),
activation='relu',
padding='same',
kernel_initializer=keras.initializers.Constant(w_and_b['conv3_1'][0]),
bias_initializer=keras.initializers.Constant(w_and_b['conv3_1'][1]),
name='conv3_1d')(x)
x = layers.Conv2D(
256, (3, 3),
activation='relu',
padding='same',
kernel_initializer=keras.initializers.Constant(w_and_b['conv3_2'][0]),
bias_initializer=keras.initializers.Constant(w_and_b['conv3_2'][1]),
name='conv3_2d')(x)
x = layers.Conv2D(
256, (3, 3),
activation='relu',
padding='same',
kernel_initializer=keras.initializers.Constant(w_and_b['conv3_3'][0]),
bias_initializer=keras.initializers.Constant(w_and_b['conv3_3'][1]),
name='conv3_3d')(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool3d')(x)
# Block 4
x = layers.Conv2D(
512, (3, 3),
activation='relu',
padding='same',
kernel_initializer=keras.initializers.Constant(w_and_b['conv4_1'][0]),
bias_initializer=keras.initializers.Constant(w_and_b['conv4_1'][1]),
name='conv4_1d')(x)
x = layers.Conv2D(
512, (3, 3),
activation='relu',
padding='same',
kernel_initializer=keras.initializers.Constant(w_and_b['conv4_2'][0]),
bias_initializer=keras.initializers.Constant(w_and_b['conv4_2'][1]),
name='conv4_2d')(x)
x = layers.Conv2D(
512, (3, 3),
activation='relu',
padding='same',
kernel_initializer=keras.initializers.Constant(w_and_b['conv4_3'][0]),
bias_initializer=keras.initializers.Constant(w_and_b['conv4_3'][1]),
name='conv4_3d')(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool4d')(x)
# Block 5
x = layers.Conv2D(
512, (3, 3),
activation='relu',
padding='same',
kernel_initializer=keras.initializers.Constant(w_and_b['conv5_1'][0]),
bias_initializer=keras.initializers.Constant(w_and_b['conv5_1'][1]),
name='conv5_1d')(x)
x = layers.Conv2D(
512, (3, 3),
activation='relu',
padding='same',
kernel_initializer=keras.initializers.Constant(w_and_b['conv5_2'][0]),
bias_initializer=keras.initializers.Constant(w_and_b['conv5_2'][1]),
name='conv5_2d')(x)
x = layers.Conv2D(
512, (3, 3),
activation='relu',
padding='same',
kernel_initializer=keras.initializers.Constant(w_and_b['conv5_3'][0]),
bias_initializer=keras.initializers.Constant(w_and_b['conv5_3'][1]),
name='conv5_3d')(x)
conv_model_depth = layers.MaxPooling2D((2, 2),
strides=(2, 2),
name='pool5d')(x)
######################
# conv_model_depth = vgg_model_depth(inputs_depth)
# fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(dropout_rgb)
# fc6_depth = layers.Dense(2048, activation='relu', name='fc6_depth')(dropout_depth)
# CONACTENATE the ends of RGB & DEPTH
merge_rgb_depth = layers.concatenate([conv_model_rgb, conv_model_depth],
axis=-1)
attention_features = cbam_block(merge_rgb_depth)
#
#
# ############ for RGB
# flat_model_rgb = layers.Flatten(name='flatten_rgb')(conv_model_rgb)
# fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(flat_model_rgb)
# dropout_rgb = layers.Dropout(0.2)(fc6_rgb)
######## for Depth
flat_model = layers.Flatten(name='flatten')(attention_features)
fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model)
bn_1 = BatchNormalization(name='1_bn')(fc6)
dropout_1 = layers.Dropout(0.5)(bn_1)
# flatten_concat = layers.Flatten(name='flatten_concat')(merge_rgb_depth)
# fc6 = layers.Dense(2048, activation='relu', name='fc6')(merge_rgb_depth)
fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1)
bn_2 = BatchNormalization(name='2_bn')(fc7)
dropout_2 = layers.Dropout(0.5)(bn_2)
#
fc8 = layers.Dense(512, activation='relu', name='fc8')(dropout_2)
bn_3 = BatchNormalization(name='3_bn')(fc8)
dropout_3 = layers.Dropout(0.5)(bn_3)
# fc9 = layers.Dense(2048, activation='relu', name='fc9')(dropout_3)
# bn_4 = BatchNormalization(name='4_bn')(fc9)
# dropout_4 = layers.Dropout(0.5)(bn_4)
#VECTORIZING OUTPUT
output = layers.Dense(n_class, activation='softmax',
name='output')(dropout_3)
# MODAL [INPUTS , OUTPUTS]
train_model = models.Model(inputs=[inputs_rgb, inputs_depth],
outputs=[output])
# weights_path = 'D:/tutorial/rgb+depth+thermal/IIITD_RGBD_new/result_multimodal_attention_bn_fold5_dropout-0.5_2fc_cont2/weights-08.h5'
# train_model.load_weights(weights_path)
train_model.summary()
for layer in train_model.layers[:-26]:
layer.trainable = False
return train_model
def VGGnet(input_shape, n_class):
"""
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:return: Keras Model used for training
"""
# x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
inputs_depth = layers.Input(shape=input_shape)
# pre_trained_model = VGGFace(include_top=False, input_shape=input_shape)
# conv_model_depth = pre_trained_model(inputs_depth)
##############################
conv_1 = layers.Conv2D(64, (11, 11),
activation='relu',
padding='same',
name='conv1_1d')(inputs_depth)
# x = layers.Conv2D(64, (3, 3), activation='relu', padding='same', name='conv1_2d')(x)
mp_1 = layers.MaxPooling2D((3, 3), strides=(2, 2), name='pool1d')(conv_1)
# Block 2
conv_2 = layers.Conv2D(128, (5, 5),
activation='relu',
padding='same',
name='conv2_1d')(mp_1)
# x = layers.Conv2D(128, (3, 3), activation='relu', padding='same', name='conv2_2d')(x)
mp_2 = layers.MaxPooling2D((3, 3), strides=(2, 2), name='pool2d')(conv_2)
# Block 3
conv_3 = layers.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='conv3')(mp_2)
conv_4 = layers.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='conv4')(conv_3)
conv_5 = layers.Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='conv5')(conv_4)
# x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_2d')(x)
# x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_3d')(x)
# mp_6 = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool3d')(conv_5)
# Block 4
# conv_7 = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_1d')(mp_6)
# x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_2d')(x)
# x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_3d')(x)
# x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool4d')(x)### skipped for custom_2
# Block 5
# x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_1d')(x)### skipped for custom_2
# x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_2d')(x)
# x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_3d')(x)
conv_model_depth = layers.MaxPooling2D((3, 3),
strides=(2, 2),
name='pool5d')(conv_5)
# pre_trained_model.summary()
#########################
# merge_layer = layers.concatenate([conv_model_depth,mp_6,mp_4])
mp1_out = layers.Conv2D(256, (4, 4),
strides=(4, 4),
activation='relu',
padding='valid',
name='mp1_out')(mp_1)
conv3_out = layers.Conv2D(256, (2, 2),
strides=(2, 2),
activation='relu',
padding='valid',
name='conv3_out')(conv_3)
# layer4_out = layers.Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='valid', name='conv5_out')(conv_5)
merge_layer = layers.concatenate([conv_model_depth, mp1_out, conv3_out])
conv6_out = layers.Conv2D(128, (1, 1),
strides=(2, 2),
activation='relu',
padding='valid',
name='conv6_out')(merge_layer)
attention_features = cbam_block(conv6_out)
flat_model_depth = layers.Flatten(name='flatten_depth')(attention_features)
# attention_features = cbam_block(flat_model_depth)
#
#
# fc6 = layers.Dense(512, activation='relu', name='fc6')(attention_features)
# dropout_1 = layers.Dropout(0.2)(fc6)
#
fc7 = layers.Dense(512, activation='relu', name='fc7')(flat_model_depth)
dropout_2 = layers.Dropout(0.2)(fc7)
# final_layer = pre_trained_model.layers[-3].output
output = layers.Dense(n_class, activation='softmax',
name='output')(dropout_2)
# Models for training and evaluation (prediction)
train_model = models.Model(inputs_depth, output)
# for layer in train_model.layers[:-6]:
# layer.trainable = False
return train_model
def VGGFace_multimodal(input_shape, n_class):
"""
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:return: Keras Model used for training
"""
# RGB MODALITY BRANCH OF CNN
inputs_rgb = layers.Input(shape=input_shape)
########################VGG/RESNET or any other network
vgg_model_rgb = VGG16(include_top=False,
weights='vggface',
input_tensor=None,
input_shape=input_shape,
pooling=None,
type_name='rgb')
conv_model_rgb = vgg_model_rgb(inputs_rgb)
########################
inputs_depth = layers.Input(shape=input_shape)
vgg_model_depth = VGG16(include_top=False,
weights='vggface',
input_tensor=None,
input_shape=input_shape,
pooling=None,
type_name='depth')
conv_model_depth = vgg_model_depth(inputs_depth)
######################
# conv_model_depth = vgg_model_depth(inputs_depth)
# fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(dropout_rgb)
# fc6_depth = layers.Dense(2048, activation='relu', name='fc6_depth')(dropout_depth)
# CONACTENATE the ends of RGB & DEPTH
merge_rgb_depth = layers.concatenate([conv_model_rgb, conv_model_depth],
axis=-1)
attention_features = cbam_block(merge_rgb_depth)
#
#
# ############ for RGB
# flat_model_rgb = layers.Flatten(name='flatten_rgb')(conv_model_rgb)
# fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(flat_model_rgb)
# dropout_rgb = layers.Dropout(0.2)(fc6_rgb)
######## for Depth
flat_model = layers.Flatten(name='flatten')(attention_features)
fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model)
bn_1 = BatchNormalization(name='1_bn')(fc6)
dropout_1 = layers.Dropout(0.5)(bn_1)
# flatten_concat = layers.Flatten(name='flatten_concat')(merge_rgb_depth)
# fc6 = layers.Dense(2048, activation='relu', name='fc6')(merge_rgb_depth)
fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1)
bn_2 = BatchNormalization(name='2_bn')(fc7)
dropout_2 = layers.Dropout(0.5)(bn_2)
# fc8 = layers.Dense(512, activation='relu', name='fc8')(dropout_2)
# bn_3 = BatchNormalization(name='3_bn')(fc8)
# dropout_3 = layers.Dropout(0.5)(bn_3)
#VECTORIZING OUTPUT
output = layers.Dense(n_class, activation='softmax',
name='output')(dropout_2)
# MODAL [INPUTS , OUTPUTS]
train_model = models.Model(inputs=[inputs_rgb, inputs_depth],
outputs=[output])
# weights_path = 'CurtinFaces/vgg_multimodal_dropout-0.5_3fc-512/weights-25.h5'
# train_model.load_weights(weights_path)
train_model.summary()
for layer in train_model.layers[:-23]:
layer.trainable = False
# for layer in train_model.layers[2].layers[:-4]:
# layer.trainable = False
# for layer in train_model.layers[3].layers[:-4]:
# layer.trainable = False
return train_model
def VGGnet(input_shape, n_class):
"""
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:return: Keras Model used for training
"""
# x = layers.Input(shape=input_shape)
pickle_in = open("vgg16_face_conv_weights_name.pkl","rb")
w_and_b = pickle.load(pickle_in)
add_weights = np.expand_dims(w_and_b['conv1_1'][0][1], axis=2)
# add_weights = np.zeros((3,3,1,64)) initiate depth channel with zeros
w_and_b['conv1_1'][0] = np.concatenate((w_and_b['conv1_1'][0],add_weights), axis = 2)
# DEPTH MODALITY BRANCH OF CNN
inputs_rgb_d = layers.Input(shape=input_shape)
#######################VGG/RESNET or any other network
# vgg_model_depth = VGGFace(include_top=False, input_shape=input_shape)
# Block 1
x = layers.Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv1_1'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv1_1'][1]), name='conv1_1d')(inputs_rgb_d)
x = layers.Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv1_2'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv1_2'][1]), name='conv1_2d')(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool1d')(x)
# Block 2
x = layers.Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv2_1'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv2_1'][1]), name='conv2_1d')(
x)
x = layers.Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv2_2'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv2_1'][1]), name='conv2_2d')(
x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool2d')(x)
# Block 3
x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv3_1'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv3_1'][1]), name='conv3_1d')(
x)
x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv3_2'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv3_2'][1]), name='conv3_2d')(
x)
x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv3_3'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv3_3'][1]), name='conv3_3d')(
x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool3d')(x)
# Block 4
x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv4_1'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv4_1'][1]), name='conv4_1d')(
x)
x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv4_2'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv4_2'][1]), name='conv4_2d')(
x)
x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv4_3'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv4_3'][1]), name='conv4_3d')(
x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool4d')(x)
# Block 5
x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv5_1'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv5_1'][1]), name='conv5_1d')(
x)
x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv5_2'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv5_2'][1]), name='conv5_2d')(
x)
x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv5_3'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv5_3'][1]), name='conv5_3d')(
x)
conv_model_rgb = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool5d')(x)
attention_features = cbam_block(conv_model_rgb, att_type='Channel')
flat_model = layers.Flatten(name='flatten')(attention_features)
fc6 = layers.Dense(1024, activation='relu', name='fc6')(flat_model)
dropout_layer_1 = layers.Dropout(0.2)(fc6)
fc7 = layers.Dense(512, activation='relu', name='fc7')(dropout_layer_1)
dropout_layer_2 = layers.Dropout(0.2)(fc7)
output = layers.Dense(n_class, activation='softmax', name='output')(dropout_layer_2)
# Models for training and evaluation (prediction)
train_model = models.Model(inputs_rgb_d, output)
# for layer in train_model.layers[:-14]:
# layer.trainable = False
return train_model
def VGGFace_multimodal(input_shape, n_class):
"""
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:return: Keras Model used for training
"""
# RGB MODALITY BRANCH OF CNN
inputs_rgb = layers.Input(shape=input_shape)
# ########################VGG/RESNET or any other network
resnet_model_rgb = RESNET50(include_top=False,
weights='vggface',
input_tensor=None,
input_shape=None,
pooling=None,
type_name='rgb')
# vgg_model_rgb = VGGFace(include_top=False, input_shape=input_shape)
conv_model_rgb = resnet_model_rgb(inputs_rgb)
#
######################## DEPTH branch
inputs_depth = layers.Input(shape=input_shape)
pre_trained_model = RESNET50(include_top=False,
weights='vggface',
input_tensor=None,
input_shape=None,
pooling=None,
type_name='depth')
# pre_trained_model = VGGFace(include_top=False, input_shape=input_shape)
# pre_trained_model.summary()
conv_model_depth = pre_trained_model(inputs_depth)
# flat_model_depth = layers.Flatten(name='flatten_depth')(conv_model_depth)
#
#
# fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model_depth)
# dropout_1 = layers.Dropout(0.2)(fc6)
#
# fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1)
# dropout_2 = layers.Dropout(0.2)(fc7)
#
## final_layer = pre_trained_model.layers[-3].output
# output = layers.Dense(20, activation='softmax', name='output')(dropout_2)
#
# # Models for training and evaluation (prediction)
# train_model = models.Model(inputs_depth, output)
# weights_path = ('D:/tutorial/rgb+depth+thermal/allData/resne_depth/weights-19.h5')
# train_model.load_weights(weights_path)
#
# final_layer = train_model.layers[-7].layers[-1].output
# resnet_model_depth = models.Model(inputs_depth,final_layer)
#
#
# conv_model_depth_1 = resnet_model_depth(inputs_depth)
merge_rgb_depth = layers.concatenate([conv_model_rgb, conv_model_depth],
axis=-1)
attention_features = cbam_block(merge_rgb_depth)
flat_model = layers.Flatten(name='flatten')(attention_features)
fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model)
dropout_1 = layers.Dropout(0.5)(fc6)
# flatten_concat = layers.Flatten(name='flatten_concat')(merge_rgb_depth)
# fc6 = layers.Dense(2048, activation='relu', name='fc6')(merge_rgb_depth)
fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1)
dropout_2 = layers.Dropout(0.5)(fc7)
#VECTORIZING OUTPUT
output = layers.Dense(n_class, activation='softmax',
name='output')(dropout_2)
# MODAL [INPUTS , OUTPUTS]
train_model = models.Model(inputs=[inputs_rgb, inputs_depth],
outputs=[output])
train_model.summary()
for layer in train_model.layers[:-19]:
layer.trainable = False
return train_model
