def VGGFace_multimodal(input_shape, n_class): """ :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :return: Keras Model used for training """ # RGB MODALITY BRANCH OF CNN inputs_rgb = layers.Input(shape=input_shape) # ########################VGG/RESNET or any other network resnet_model_rgb = RESNET50(include_top=False, weights='vggface', input_tensor=None, input_shape=None, pooling=None, type_name='rgb') conv_model_rgb = resnet_model_rgb(inputs_rgb) # ######################## inputs_depth = layers.Input(shape=input_shape) resnet_model_depth = RESNET50(include_top=False, weights='vggface', input_tensor=None, input_shape=None, pooling=None, type_name='depth') # pre_trained_model = VGGFace(include_top=False, input_shape=input_shape) # pre_trained_model.summary() conv_model_depth = resnet_model_depth(inputs_depth) merge_rgb_depth = layers.concatenate([conv_model_rgb, conv_model_depth], axis=-1) attention_features = cbam_block(merge_rgb_depth) flat_model = layers.Flatten(name='flatten')(attention_features) fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model) dropout_1 = layers.Dropout(0.2)(fc6) # flatten_concat = layers.Flatten(name='flatten_concat')(merge_rgb_depth) # fc6 = layers.Dense(2048, activation='relu', name='fc6')(merge_rgb_depth) fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1) dropout_2 = layers.Dropout(0.2)(fc7) #VECTORIZING OUTPUT output = layers.Dense(n_class, activation='softmax', name='output')(dropout_2) # MODAL [INPUTS , OUTPUTS] train_model = models.Model(inputs=[inputs_rgb, inputs_depth], outputs=[output]) train_model.summary() for layer in train_model.layers[:-19]: layer.trainable = False return train_model
def sc_block(input_map): block_conv1 = keras.layers.Conv2D(64, (3, 3), padding='same')(input_map) block_bn1 = keras.layers.BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001)(block_conv1) block_Lrelu1 = keras.layers.LeakyReLU(alpha=0.05)(block_bn1) block_concat1 = keras.layers.concatenate([input_map, block_Lrelu1], axis=-1) block_conv2 = keras.layers.Conv2D(64, (3, 3), padding='same')(block_concat1) block_sc = cbam_block(block_conv2) block_concat2 = keras.layers.concatenate([input_map, block_sc], axis=-1) return block_concat2
def VGGFace_multimodal(input_shape, n_class): """ :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :return: Keras Model used for training """ # RGB MODALITY BRANCH OF CNN ###################### # conv_model_depth = vgg_model_depth(inputs_depth) # fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(dropout_rgb) # fc6_depth = layers.Dense(2048, activation='relu', name='fc6_depth')(dropout_depth) # CONACTENATE the ends of RGB & DEPTH alpha = 0.7 merge_rgb_depth = layers.concatenate([conv_model_rgb, conv_model_depth], axis=-1) attention_features = cbam_block(merge_rgb_depth) # # # ############ for RGB # flat_model_rgb = layers.Flatten(name='flatten_rgb')(conv_model_rgb) # fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(flat_model_rgb) # dropout_rgb = layers.Dropout(0.2)(fc6_rgb) ######## for Depth flat_model = layers.Flatten(name='flatten')(attention_features) fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model) dropout_1 = layers.Dropout(0.2)(fc6) # flatten_concat = layers.Flatten(name='flatten_concat')(merge_rgb_depth) # fc6 = layers.Dense(2048, activation='relu', name='fc6')(merge_rgb_depth) fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1) dropout_2 = layers.Dropout(0.2)(fc7) #VECTORIZING OUTPUT output = layers.Dense(n_class, activation='softmax', name='output')(dropout_2) # MODAL [INPUTS , OUTPUTS] train_model = models.Model(inputs=[inputs_rgb, inputs_depth], outputs=[output]) train_model.summary() for layer in train_model.layers[:-19]: layer.trainable = False return train_model
def VGGFace_multimodal(input_shape, n_class): """ :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :return: Keras Model used for training """ # RGB MODALITY BRANCH OF CNN inputs_rgb = layers.Input(shape=input_shape) ########################VGG/RESNET or any other network vgg_model_rgb = VGGFace(include_top=False, input_shape=input_shape) ######################## conv_model_rgb = vgg_model_rgb(inputs_rgb) ## get weights pickle_in = open("vgg16_face_conv_weights_name.pkl", "rb") w_and_b = pickle.load(pickle_in) # DEPTH MODALITY BRANCH OF CNN inputs_depth = layers.Input(shape=input_shape) #######################VGG/RESNET or any other network # vgg_model_depth = VGGFace(include_top=False, input_shape=input_shape) # Block 1 x = layers.Conv2D( 64, (3, 3), activation='relu', padding='same', kernel_initializer=keras.initializers.Constant(w_and_b['conv1_1'][0]), bias_initializer=keras.initializers.Constant(w_and_b['conv1_1'][1]), name='conv1_1d')(inputs_depth) x = layers.Conv2D( 64, (3, 3), activation='relu', padding='same', kernel_initializer=keras.initializers.Constant(w_and_b['conv1_2'][0]), bias_initializer=keras.initializers.Constant(w_and_b['conv1_2'][1]), name='conv1_2d')(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool1d')(x) # Block 2 x = layers.Conv2D( 128, (3, 3), activation='relu', padding='same', kernel_initializer=keras.initializers.Constant(w_and_b['conv2_1'][0]), bias_initializer=keras.initializers.Constant(w_and_b['conv2_1'][1]), name='conv2_1d')(x) x = layers.Conv2D( 128, (3, 3), activation='relu', padding='same', kernel_initializer=keras.initializers.Constant(w_and_b['conv2_2'][0]), bias_initializer=keras.initializers.Constant(w_and_b['conv2_1'][1]), name='conv2_2d')(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool2d')(x) # Block 3 x = layers.Conv2D( 256, (3, 3), activation='relu', padding='same', kernel_initializer=keras.initializers.Constant(w_and_b['conv3_1'][0]), bias_initializer=keras.initializers.Constant(w_and_b['conv3_1'][1]), name='conv3_1d')(x) x = layers.Conv2D( 256, (3, 3), activation='relu', padding='same', kernel_initializer=keras.initializers.Constant(w_and_b['conv3_2'][0]), bias_initializer=keras.initializers.Constant(w_and_b['conv3_2'][1]), name='conv3_2d')(x) x = layers.Conv2D( 256, (3, 3), activation='relu', padding='same', kernel_initializer=keras.initializers.Constant(w_and_b['conv3_3'][0]), bias_initializer=keras.initializers.Constant(w_and_b['conv3_3'][1]), name='conv3_3d')(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool3d')(x) # Block 4 x = layers.Conv2D( 512, (3, 3), activation='relu', padding='same', kernel_initializer=keras.initializers.Constant(w_and_b['conv4_1'][0]), bias_initializer=keras.initializers.Constant(w_and_b['conv4_1'][1]), name='conv4_1d')(x) x = layers.Conv2D( 512, (3, 3), activation='relu', padding='same', kernel_initializer=keras.initializers.Constant(w_and_b['conv4_2'][0]), bias_initializer=keras.initializers.Constant(w_and_b['conv4_2'][1]), name='conv4_2d')(x) x = layers.Conv2D( 512, (3, 3), activation='relu', padding='same', kernel_initializer=keras.initializers.Constant(w_and_b['conv4_3'][0]), bias_initializer=keras.initializers.Constant(w_and_b['conv4_3'][1]), name='conv4_3d')(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool4d')(x) # Block 5 x = layers.Conv2D( 512, (3, 3), activation='relu', padding='same', kernel_initializer=keras.initializers.Constant(w_and_b['conv5_1'][0]), bias_initializer=keras.initializers.Constant(w_and_b['conv5_1'][1]), name='conv5_1d')(x) x = layers.Conv2D( 512, (3, 3), activation='relu', padding='same', kernel_initializer=keras.initializers.Constant(w_and_b['conv5_2'][0]), bias_initializer=keras.initializers.Constant(w_and_b['conv5_2'][1]), name='conv5_2d')(x) x = layers.Conv2D( 512, (3, 3), activation='relu', padding='same', kernel_initializer=keras.initializers.Constant(w_and_b['conv5_3'][0]), bias_initializer=keras.initializers.Constant(w_and_b['conv5_3'][1]), name='conv5_3d')(x) conv_model_depth = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool5d')(x) ###################### # conv_model_depth = vgg_model_depth(inputs_depth) # fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(dropout_rgb) # fc6_depth = layers.Dense(2048, activation='relu', name='fc6_depth')(dropout_depth) # CONACTENATE the ends of RGB & DEPTH merge_rgb_depth = layers.concatenate([conv_model_rgb, conv_model_depth], axis=-1) attention_features = cbam_block(merge_rgb_depth) # # # ############ for RGB # flat_model_rgb = layers.Flatten(name='flatten_rgb')(conv_model_rgb) # fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(flat_model_rgb) # dropout_rgb = layers.Dropout(0.2)(fc6_rgb) ######## for Depth flat_model = layers.Flatten(name='flatten')(attention_features) fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model) bn_1 = BatchNormalization(name='1_bn')(fc6) dropout_1 = layers.Dropout(0.5)(bn_1) # flatten_concat = layers.Flatten(name='flatten_concat')(merge_rgb_depth) # fc6 = layers.Dense(2048, activation='relu', name='fc6')(merge_rgb_depth) fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1) bn_2 = BatchNormalization(name='2_bn')(fc7) dropout_2 = layers.Dropout(0.5)(bn_2) # fc8 = layers.Dense(512, activation='relu', name='fc8')(dropout_2) bn_3 = BatchNormalization(name='3_bn')(fc8) dropout_3 = layers.Dropout(0.5)(bn_3) # fc9 = layers.Dense(2048, activation='relu', name='fc9')(dropout_3) # bn_4 = BatchNormalization(name='4_bn')(fc9) # dropout_4 = layers.Dropout(0.5)(bn_4) #VECTORIZING OUTPUT output = layers.Dense(n_class, activation='softmax', name='output')(dropout_3) # MODAL [INPUTS , OUTPUTS] train_model = models.Model(inputs=[inputs_rgb, inputs_depth], outputs=[output]) # weights_path = 'D:/tutorial/rgb+depth+thermal/IIITD_RGBD_new/result_multimodal_attention_bn_fold5_dropout-0.5_2fc_cont2/weights-08.h5' # train_model.load_weights(weights_path) train_model.summary() for layer in train_model.layers[:-26]: layer.trainable = False return train_model
def VGGnet(input_shape, n_class): """ :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :return: Keras Model used for training """ # x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer inputs_depth = layers.Input(shape=input_shape) # pre_trained_model = VGGFace(include_top=False, input_shape=input_shape) # conv_model_depth = pre_trained_model(inputs_depth) ############################## conv_1 = layers.Conv2D(64, (11, 11), activation='relu', padding='same', name='conv1_1d')(inputs_depth) # x = layers.Conv2D(64, (3, 3), activation='relu', padding='same', name='conv1_2d')(x) mp_1 = layers.MaxPooling2D((3, 3), strides=(2, 2), name='pool1d')(conv_1) # Block 2 conv_2 = layers.Conv2D(128, (5, 5), activation='relu', padding='same', name='conv2_1d')(mp_1) # x = layers.Conv2D(128, (3, 3), activation='relu', padding='same', name='conv2_2d')(x) mp_2 = layers.MaxPooling2D((3, 3), strides=(2, 2), name='pool2d')(conv_2) # Block 3 conv_3 = layers.Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3')(mp_2) conv_4 = layers.Conv2D(256, (3, 3), activation='relu', padding='same', name='conv4')(conv_3) conv_5 = layers.Conv2D(256, (3, 3), activation='relu', padding='same', name='conv5')(conv_4) # x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_2d')(x) # x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_3d')(x) # mp_6 = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool3d')(conv_5) # Block 4 # conv_7 = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_1d')(mp_6) # x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_2d')(x) # x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_3d')(x) # x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool4d')(x)### skipped for custom_2 # Block 5 # x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_1d')(x)### skipped for custom_2 # x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_2d')(x) # x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_3d')(x) conv_model_depth = layers.MaxPooling2D((3, 3), strides=(2, 2), name='pool5d')(conv_5) # pre_trained_model.summary() ######################### # merge_layer = layers.concatenate([conv_model_depth,mp_6,mp_4]) mp1_out = layers.Conv2D(256, (4, 4), strides=(4, 4), activation='relu', padding='valid', name='mp1_out')(mp_1) conv3_out = layers.Conv2D(256, (2, 2), strides=(2, 2), activation='relu', padding='valid', name='conv3_out')(conv_3) # layer4_out = layers.Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='valid', name='conv5_out')(conv_5) merge_layer = layers.concatenate([conv_model_depth, mp1_out, conv3_out]) conv6_out = layers.Conv2D(128, (1, 1), strides=(2, 2), activation='relu', padding='valid', name='conv6_out')(merge_layer) attention_features = cbam_block(conv6_out) flat_model_depth = layers.Flatten(name='flatten_depth')(attention_features) # attention_features = cbam_block(flat_model_depth) # # # fc6 = layers.Dense(512, activation='relu', name='fc6')(attention_features) # dropout_1 = layers.Dropout(0.2)(fc6) # fc7 = layers.Dense(512, activation='relu', name='fc7')(flat_model_depth) dropout_2 = layers.Dropout(0.2)(fc7) # final_layer = pre_trained_model.layers[-3].output output = layers.Dense(n_class, activation='softmax', name='output')(dropout_2) # Models for training and evaluation (prediction) train_model = models.Model(inputs_depth, output) # for layer in train_model.layers[:-6]: # layer.trainable = False return train_model
def VGGFace_multimodal(input_shape, n_class): """ :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :return: Keras Model used for training """ # RGB MODALITY BRANCH OF CNN inputs_rgb = layers.Input(shape=input_shape) ########################VGG/RESNET or any other network vgg_model_rgb = VGG16(include_top=False, weights='vggface', input_tensor=None, input_shape=input_shape, pooling=None, type_name='rgb') conv_model_rgb = vgg_model_rgb(inputs_rgb) ######################## inputs_depth = layers.Input(shape=input_shape) vgg_model_depth = VGG16(include_top=False, weights='vggface', input_tensor=None, input_shape=input_shape, pooling=None, type_name='depth') conv_model_depth = vgg_model_depth(inputs_depth) ###################### # conv_model_depth = vgg_model_depth(inputs_depth) # fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(dropout_rgb) # fc6_depth = layers.Dense(2048, activation='relu', name='fc6_depth')(dropout_depth) # CONACTENATE the ends of RGB & DEPTH merge_rgb_depth = layers.concatenate([conv_model_rgb, conv_model_depth], axis=-1) attention_features = cbam_block(merge_rgb_depth) # # # ############ for RGB # flat_model_rgb = layers.Flatten(name='flatten_rgb')(conv_model_rgb) # fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(flat_model_rgb) # dropout_rgb = layers.Dropout(0.2)(fc6_rgb) ######## for Depth flat_model = layers.Flatten(name='flatten')(attention_features) fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model) bn_1 = BatchNormalization(name='1_bn')(fc6) dropout_1 = layers.Dropout(0.5)(bn_1) # flatten_concat = layers.Flatten(name='flatten_concat')(merge_rgb_depth) # fc6 = layers.Dense(2048, activation='relu', name='fc6')(merge_rgb_depth) fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1) bn_2 = BatchNormalization(name='2_bn')(fc7) dropout_2 = layers.Dropout(0.5)(bn_2) # fc8 = layers.Dense(512, activation='relu', name='fc8')(dropout_2) # bn_3 = BatchNormalization(name='3_bn')(fc8) # dropout_3 = layers.Dropout(0.5)(bn_3) #VECTORIZING OUTPUT output = layers.Dense(n_class, activation='softmax', name='output')(dropout_2) # MODAL [INPUTS , OUTPUTS] train_model = models.Model(inputs=[inputs_rgb, inputs_depth], outputs=[output]) # weights_path = 'CurtinFaces/vgg_multimodal_dropout-0.5_3fc-512/weights-25.h5' # train_model.load_weights(weights_path) train_model.summary() for layer in train_model.layers[:-23]: layer.trainable = False # for layer in train_model.layers[2].layers[:-4]: # layer.trainable = False # for layer in train_model.layers[3].layers[:-4]: # layer.trainable = False return train_model
def VGGnet(input_shape, n_class): """ :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :return: Keras Model used for training """ # x = layers.Input(shape=input_shape) pickle_in = open("vgg16_face_conv_weights_name.pkl","rb") w_and_b = pickle.load(pickle_in) add_weights = np.expand_dims(w_and_b['conv1_1'][0][1], axis=2) # add_weights = np.zeros((3,3,1,64)) initiate depth channel with zeros w_and_b['conv1_1'][0] = np.concatenate((w_and_b['conv1_1'][0],add_weights), axis = 2) # DEPTH MODALITY BRANCH OF CNN inputs_rgb_d = layers.Input(shape=input_shape) #######################VGG/RESNET or any other network # vgg_model_depth = VGGFace(include_top=False, input_shape=input_shape) # Block 1 x = layers.Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv1_1'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv1_1'][1]), name='conv1_1d')(inputs_rgb_d) x = layers.Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv1_2'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv1_2'][1]), name='conv1_2d')(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool1d')(x) # Block 2 x = layers.Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv2_1'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv2_1'][1]), name='conv2_1d')( x) x = layers.Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv2_2'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv2_1'][1]), name='conv2_2d')( x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool2d')(x) # Block 3 x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv3_1'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv3_1'][1]), name='conv3_1d')( x) x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv3_2'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv3_2'][1]), name='conv3_2d')( x) x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv3_3'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv3_3'][1]), name='conv3_3d')( x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool3d')(x) # Block 4 x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv4_1'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv4_1'][1]), name='conv4_1d')( x) x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv4_2'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv4_2'][1]), name='conv4_2d')( x) x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv4_3'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv4_3'][1]), name='conv4_3d')( x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool4d')(x) # Block 5 x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv5_1'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv5_1'][1]), name='conv5_1d')( x) x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv5_2'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv5_2'][1]), name='conv5_2d')( x) x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv5_3'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv5_3'][1]), name='conv5_3d')( x) conv_model_rgb = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool5d')(x) attention_features = cbam_block(conv_model_rgb, att_type='Channel') flat_model = layers.Flatten(name='flatten')(attention_features) fc6 = layers.Dense(1024, activation='relu', name='fc6')(flat_model) dropout_layer_1 = layers.Dropout(0.2)(fc6) fc7 = layers.Dense(512, activation='relu', name='fc7')(dropout_layer_1) dropout_layer_2 = layers.Dropout(0.2)(fc7) output = layers.Dense(n_class, activation='softmax', name='output')(dropout_layer_2) # Models for training and evaluation (prediction) train_model = models.Model(inputs_rgb_d, output) # for layer in train_model.layers[:-14]: # layer.trainable = False return train_model
def VGGFace_multimodal(input_shape, n_class): """ :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :return: Keras Model used for training """ # RGB MODALITY BRANCH OF CNN inputs_rgb = layers.Input(shape=input_shape) # ########################VGG/RESNET or any other network resnet_model_rgb = RESNET50(include_top=False, weights='vggface', input_tensor=None, input_shape=None, pooling=None, type_name='rgb') # vgg_model_rgb = VGGFace(include_top=False, input_shape=input_shape) conv_model_rgb = resnet_model_rgb(inputs_rgb) # ######################## DEPTH branch inputs_depth = layers.Input(shape=input_shape) pre_trained_model = RESNET50(include_top=False, weights='vggface', input_tensor=None, input_shape=None, pooling=None, type_name='depth') # pre_trained_model = VGGFace(include_top=False, input_shape=input_shape) # pre_trained_model.summary() conv_model_depth = pre_trained_model(inputs_depth) # flat_model_depth = layers.Flatten(name='flatten_depth')(conv_model_depth) # # # fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model_depth) # dropout_1 = layers.Dropout(0.2)(fc6) # # fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1) # dropout_2 = layers.Dropout(0.2)(fc7) # ## final_layer = pre_trained_model.layers[-3].output # output = layers.Dense(20, activation='softmax', name='output')(dropout_2) # # # Models for training and evaluation (prediction) # train_model = models.Model(inputs_depth, output) # weights_path = ('D:/tutorial/rgb+depth+thermal/allData/resne_depth/weights-19.h5') # train_model.load_weights(weights_path) # # final_layer = train_model.layers[-7].layers[-1].output # resnet_model_depth = models.Model(inputs_depth,final_layer) # # # conv_model_depth_1 = resnet_model_depth(inputs_depth) merge_rgb_depth = layers.concatenate([conv_model_rgb, conv_model_depth], axis=-1) attention_features = cbam_block(merge_rgb_depth) flat_model = layers.Flatten(name='flatten')(attention_features) fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model) dropout_1 = layers.Dropout(0.5)(fc6) # flatten_concat = layers.Flatten(name='flatten_concat')(merge_rgb_depth) # fc6 = layers.Dense(2048, activation='relu', name='fc6')(merge_rgb_depth) fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1) dropout_2 = layers.Dropout(0.5)(fc7) #VECTORIZING OUTPUT output = layers.Dense(n_class, activation='softmax', name='output')(dropout_2) # MODAL [INPUTS , OUTPUTS] train_model = models.Model(inputs=[inputs_rgb, inputs_depth], outputs=[output]) train_model.summary() for layer in train_model.layers[:-19]: layer.trainable = False return train_model