コード例 #1
0
def VGGFace_multimodal(input_shape, n_class):
    """
    
    :param input_shape: data shape, 3d, [width, height, channels]
    :param n_class: number of classes
    
    :return:  Keras Model used for training
    """
    # RGB MODALITY BRANCH OF CNN
    inputs_rgb = layers.Input(shape=input_shape)
    #    ########################VGG/RESNET or any other network
    resnet_model_rgb = RESNET50(include_top=False,
                                weights='vggface',
                                input_tensor=None,
                                input_shape=None,
                                pooling=None,
                                type_name='rgb')
    conv_model_rgb = resnet_model_rgb(inputs_rgb)
    #
    ########################

    inputs_depth = layers.Input(shape=input_shape)
    resnet_model_depth = RESNET50(include_top=False,
                                  weights='vggface',
                                  input_tensor=None,
                                  input_shape=None,
                                  pooling=None,
                                  type_name='depth')
    #    pre_trained_model = VGGFace(include_top=False, input_shape=input_shape)
    #    pre_trained_model.summary()
    conv_model_depth = resnet_model_depth(inputs_depth)

    merge_rgb_depth = layers.concatenate([conv_model_rgb, conv_model_depth],
                                         axis=-1)
    attention_features = cbam_block(merge_rgb_depth)

    flat_model = layers.Flatten(name='flatten')(attention_features)
    fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model)
    dropout_1 = layers.Dropout(0.2)(fc6)

    #    flatten_concat = layers.Flatten(name='flatten_concat')(merge_rgb_depth)
    #    fc6 = layers.Dense(2048, activation='relu', name='fc6')(merge_rgb_depth)
    fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1)
    dropout_2 = layers.Dropout(0.2)(fc7)

    #VECTORIZING OUTPUT
    output = layers.Dense(n_class, activation='softmax',
                          name='output')(dropout_2)

    # MODAL [INPUTS , OUTPUTS]
    train_model = models.Model(inputs=[inputs_rgb, inputs_depth],
                               outputs=[output])

    train_model.summary()
    for layer in train_model.layers[:-19]:
        layer.trainable = False

    return train_model
コード例 #2
0
def sc_block(input_map):
    block_conv1 = keras.layers.Conv2D(64, (3, 3), padding='same')(input_map)
    block_bn1 = keras.layers.BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001)(block_conv1)
    block_Lrelu1 = keras.layers.LeakyReLU(alpha=0.05)(block_bn1)
    block_concat1 = keras.layers.concatenate([input_map, block_Lrelu1], axis=-1)
    block_conv2 = keras.layers.Conv2D(64, (3, 3), padding='same')(block_concat1)
    block_sc = cbam_block(block_conv2)
    block_concat2 = keras.layers.concatenate([input_map, block_sc], axis=-1)
    return block_concat2
コード例 #3
0
def VGGFace_multimodal(input_shape, n_class):
    """
    
    :param input_shape: data shape, 3d, [width, height, channels]
    :param n_class: number of classes
    
    :return:  Keras Model used for training
    """
    # RGB MODALITY BRANCH OF CNN

    ######################
    #    conv_model_depth = vgg_model_depth(inputs_depth)

    #    fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(dropout_rgb)
    #    fc6_depth = layers.Dense(2048, activation='relu', name='fc6_depth')(dropout_depth)

    # CONACTENATE the ends of RGB & DEPTH
    alpha = 0.7
    merge_rgb_depth = layers.concatenate([conv_model_rgb, conv_model_depth],
                                         axis=-1)
    attention_features = cbam_block(merge_rgb_depth)
    #

    #
    #    ############ for RGB
    #    flat_model_rgb = layers.Flatten(name='flatten_rgb')(conv_model_rgb)
    #    fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(flat_model_rgb)
    #    dropout_rgb = layers.Dropout(0.2)(fc6_rgb)

    ######## for Depth
    flat_model = layers.Flatten(name='flatten')(attention_features)
    fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model)
    dropout_1 = layers.Dropout(0.2)(fc6)

    #    flatten_concat = layers.Flatten(name='flatten_concat')(merge_rgb_depth)
    #    fc6 = layers.Dense(2048, activation='relu', name='fc6')(merge_rgb_depth)
    fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1)
    dropout_2 = layers.Dropout(0.2)(fc7)

    #VECTORIZING OUTPUT
    output = layers.Dense(n_class, activation='softmax',
                          name='output')(dropout_2)

    # MODAL [INPUTS , OUTPUTS]
    train_model = models.Model(inputs=[inputs_rgb, inputs_depth],
                               outputs=[output])

    train_model.summary()
    for layer in train_model.layers[:-19]:
        layer.trainable = False

    return train_model
def VGGFace_multimodal(input_shape, n_class):
    """
    
    :param input_shape: data shape, 3d, [width, height, channels]
    :param n_class: number of classes
    
    :return:  Keras Model used for training
    """
    # RGB MODALITY BRANCH OF CNN
    inputs_rgb = layers.Input(shape=input_shape)
    ########################VGG/RESNET or any other network
    vgg_model_rgb = VGGFace(include_top=False, input_shape=input_shape)

    ########################
    conv_model_rgb = vgg_model_rgb(inputs_rgb)

    ## get weights
    pickle_in = open("vgg16_face_conv_weights_name.pkl", "rb")
    w_and_b = pickle.load(pickle_in)
    # DEPTH MODALITY BRANCH OF CNN
    inputs_depth = layers.Input(shape=input_shape)
    #######################VGG/RESNET or any other network
    #    vgg_model_depth = VGGFace(include_top=False, input_shape=input_shape)
    # Block 1
    x = layers.Conv2D(
        64, (3, 3),
        activation='relu',
        padding='same',
        kernel_initializer=keras.initializers.Constant(w_and_b['conv1_1'][0]),
        bias_initializer=keras.initializers.Constant(w_and_b['conv1_1'][1]),
        name='conv1_1d')(inputs_depth)
    x = layers.Conv2D(
        64, (3, 3),
        activation='relu',
        padding='same',
        kernel_initializer=keras.initializers.Constant(w_and_b['conv1_2'][0]),
        bias_initializer=keras.initializers.Constant(w_and_b['conv1_2'][1]),
        name='conv1_2d')(x)
    x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool1d')(x)

    # Block 2
    x = layers.Conv2D(
        128, (3, 3),
        activation='relu',
        padding='same',
        kernel_initializer=keras.initializers.Constant(w_and_b['conv2_1'][0]),
        bias_initializer=keras.initializers.Constant(w_and_b['conv2_1'][1]),
        name='conv2_1d')(x)
    x = layers.Conv2D(
        128, (3, 3),
        activation='relu',
        padding='same',
        kernel_initializer=keras.initializers.Constant(w_and_b['conv2_2'][0]),
        bias_initializer=keras.initializers.Constant(w_and_b['conv2_1'][1]),
        name='conv2_2d')(x)
    x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool2d')(x)

    # Block 3
    x = layers.Conv2D(
        256, (3, 3),
        activation='relu',
        padding='same',
        kernel_initializer=keras.initializers.Constant(w_and_b['conv3_1'][0]),
        bias_initializer=keras.initializers.Constant(w_and_b['conv3_1'][1]),
        name='conv3_1d')(x)
    x = layers.Conv2D(
        256, (3, 3),
        activation='relu',
        padding='same',
        kernel_initializer=keras.initializers.Constant(w_and_b['conv3_2'][0]),
        bias_initializer=keras.initializers.Constant(w_and_b['conv3_2'][1]),
        name='conv3_2d')(x)
    x = layers.Conv2D(
        256, (3, 3),
        activation='relu',
        padding='same',
        kernel_initializer=keras.initializers.Constant(w_and_b['conv3_3'][0]),
        bias_initializer=keras.initializers.Constant(w_and_b['conv3_3'][1]),
        name='conv3_3d')(x)
    x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool3d')(x)

    # Block 4
    x = layers.Conv2D(
        512, (3, 3),
        activation='relu',
        padding='same',
        kernel_initializer=keras.initializers.Constant(w_and_b['conv4_1'][0]),
        bias_initializer=keras.initializers.Constant(w_and_b['conv4_1'][1]),
        name='conv4_1d')(x)
    x = layers.Conv2D(
        512, (3, 3),
        activation='relu',
        padding='same',
        kernel_initializer=keras.initializers.Constant(w_and_b['conv4_2'][0]),
        bias_initializer=keras.initializers.Constant(w_and_b['conv4_2'][1]),
        name='conv4_2d')(x)
    x = layers.Conv2D(
        512, (3, 3),
        activation='relu',
        padding='same',
        kernel_initializer=keras.initializers.Constant(w_and_b['conv4_3'][0]),
        bias_initializer=keras.initializers.Constant(w_and_b['conv4_3'][1]),
        name='conv4_3d')(x)
    x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool4d')(x)

    # Block 5
    x = layers.Conv2D(
        512, (3, 3),
        activation='relu',
        padding='same',
        kernel_initializer=keras.initializers.Constant(w_and_b['conv5_1'][0]),
        bias_initializer=keras.initializers.Constant(w_and_b['conv5_1'][1]),
        name='conv5_1d')(x)
    x = layers.Conv2D(
        512, (3, 3),
        activation='relu',
        padding='same',
        kernel_initializer=keras.initializers.Constant(w_and_b['conv5_2'][0]),
        bias_initializer=keras.initializers.Constant(w_and_b['conv5_2'][1]),
        name='conv5_2d')(x)
    x = layers.Conv2D(
        512, (3, 3),
        activation='relu',
        padding='same',
        kernel_initializer=keras.initializers.Constant(w_and_b['conv5_3'][0]),
        bias_initializer=keras.initializers.Constant(w_and_b['conv5_3'][1]),
        name='conv5_3d')(x)
    conv_model_depth = layers.MaxPooling2D((2, 2),
                                           strides=(2, 2),
                                           name='pool5d')(x)
    ######################
    #    conv_model_depth = vgg_model_depth(inputs_depth)

    #    fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(dropout_rgb)
    #    fc6_depth = layers.Dense(2048, activation='relu', name='fc6_depth')(dropout_depth)

    # CONACTENATE the ends of RGB & DEPTH

    merge_rgb_depth = layers.concatenate([conv_model_rgb, conv_model_depth],
                                         axis=-1)
    attention_features = cbam_block(merge_rgb_depth)
    #

    #
    #    ############ for RGB
    #    flat_model_rgb = layers.Flatten(name='flatten_rgb')(conv_model_rgb)
    #    fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(flat_model_rgb)
    #    dropout_rgb = layers.Dropout(0.2)(fc6_rgb)

    ######## for Depth
    flat_model = layers.Flatten(name='flatten')(attention_features)
    fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model)
    bn_1 = BatchNormalization(name='1_bn')(fc6)
    dropout_1 = layers.Dropout(0.5)(bn_1)

    #    flatten_concat = layers.Flatten(name='flatten_concat')(merge_rgb_depth)
    #    fc6 = layers.Dense(2048, activation='relu', name='fc6')(merge_rgb_depth)
    fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1)
    bn_2 = BatchNormalization(name='2_bn')(fc7)
    dropout_2 = layers.Dropout(0.5)(bn_2)

    #
    fc8 = layers.Dense(512, activation='relu', name='fc8')(dropout_2)
    bn_3 = BatchNormalization(name='3_bn')(fc8)
    dropout_3 = layers.Dropout(0.5)(bn_3)

    #    fc9 = layers.Dense(2048, activation='relu', name='fc9')(dropout_3)
    #    bn_4 = BatchNormalization(name='4_bn')(fc9)
    #    dropout_4 = layers.Dropout(0.5)(bn_4)

    #VECTORIZING OUTPUT
    output = layers.Dense(n_class, activation='softmax',
                          name='output')(dropout_3)

    # MODAL [INPUTS , OUTPUTS]
    train_model = models.Model(inputs=[inputs_rgb, inputs_depth],
                               outputs=[output])
    #    weights_path = 'D:/tutorial/rgb+depth+thermal/IIITD_RGBD_new/result_multimodal_attention_bn_fold5_dropout-0.5_2fc_cont2/weights-08.h5'
    #    train_model.load_weights(weights_path)

    train_model.summary()
    for layer in train_model.layers[:-26]:
        layer.trainable = False

    return train_model
コード例 #5
0
def VGGnet(input_shape, n_class):
    """
    
    :param input_shape: data shape, 3d, [width, height, channels]
    :param n_class: number of classes
    
    :return:  Keras Model used for training
    """
    #    x = layers.Input(shape=input_shape)

    # Layer 1: Just a conventional Conv2D layer
    inputs_depth = layers.Input(shape=input_shape)
    #    pre_trained_model = VGGFace(include_top=False, input_shape=input_shape)
    #    conv_model_depth = pre_trained_model(inputs_depth)
    ##############################
    conv_1 = layers.Conv2D(64, (11, 11),
                           activation='relu',
                           padding='same',
                           name='conv1_1d')(inputs_depth)
    #    x = layers.Conv2D(64, (3, 3), activation='relu', padding='same', name='conv1_2d')(x)
    mp_1 = layers.MaxPooling2D((3, 3), strides=(2, 2), name='pool1d')(conv_1)

    # Block 2
    conv_2 = layers.Conv2D(128, (5, 5),
                           activation='relu',
                           padding='same',
                           name='conv2_1d')(mp_1)
    #    x = layers.Conv2D(128, (3, 3), activation='relu', padding='same', name='conv2_2d')(x)
    mp_2 = layers.MaxPooling2D((3, 3), strides=(2, 2), name='pool2d')(conv_2)

    # Block 3
    conv_3 = layers.Conv2D(256, (3, 3),
                           activation='relu',
                           padding='same',
                           name='conv3')(mp_2)
    conv_4 = layers.Conv2D(256, (3, 3),
                           activation='relu',
                           padding='same',
                           name='conv4')(conv_3)
    conv_5 = layers.Conv2D(256, (3, 3),
                           activation='relu',
                           padding='same',
                           name='conv5')(conv_4)
    #    x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_2d')(x)
    #    x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_3d')(x)
    #    mp_6 = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool3d')(conv_5)

    # Block 4
    #    conv_7 = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_1d')(mp_6)
    #    x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_2d')(x)
    #    x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_3d')(x)
    #    x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool4d')(x)### skipped for custom_2

    # Block 5
    #    x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_1d')(x)### skipped for custom_2
    #    x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_2d')(x)
    #    x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_3d')(x)
    conv_model_depth = layers.MaxPooling2D((3, 3),
                                           strides=(2, 2),
                                           name='pool5d')(conv_5)
    #    pre_trained_model.summary()
    #########################

    #    merge_layer = layers.concatenate([conv_model_depth,mp_6,mp_4])
    mp1_out = layers.Conv2D(256, (4, 4),
                            strides=(4, 4),
                            activation='relu',
                            padding='valid',
                            name='mp1_out')(mp_1)
    conv3_out = layers.Conv2D(256, (2, 2),
                              strides=(2, 2),
                              activation='relu',
                              padding='valid',
                              name='conv3_out')(conv_3)
    #    layer4_out = layers.Conv2D(256, (3, 3), strides=(2, 2), activation='relu', padding='valid', name='conv5_out')(conv_5)
    merge_layer = layers.concatenate([conv_model_depth, mp1_out, conv3_out])
    conv6_out = layers.Conv2D(128, (1, 1),
                              strides=(2, 2),
                              activation='relu',
                              padding='valid',
                              name='conv6_out')(merge_layer)
    attention_features = cbam_block(conv6_out)
    flat_model_depth = layers.Flatten(name='flatten_depth')(attention_features)
    #    attention_features = cbam_block(flat_model_depth)
    #
    #
    #    fc6 = layers.Dense(512, activation='relu', name='fc6')(attention_features)
    #    dropout_1 = layers.Dropout(0.2)(fc6)
    #
    fc7 = layers.Dense(512, activation='relu', name='fc7')(flat_model_depth)
    dropout_2 = layers.Dropout(0.2)(fc7)

    #    final_layer = pre_trained_model.layers[-3].output
    output = layers.Dense(n_class, activation='softmax',
                          name='output')(dropout_2)

    # Models for training and evaluation (prediction)
    train_model = models.Model(inputs_depth, output)
    #    for layer in train_model.layers[:-6]:
    #        layer.trainable = False

    return train_model
def VGGFace_multimodal(input_shape, n_class):
    """
    
    :param input_shape: data shape, 3d, [width, height, channels]
    :param n_class: number of classes
    
    :return:  Keras Model used for training
    """
    # RGB MODALITY BRANCH OF CNN
    inputs_rgb = layers.Input(shape=input_shape)
    ########################VGG/RESNET or any other network
    vgg_model_rgb = VGG16(include_top=False,
                          weights='vggface',
                          input_tensor=None,
                          input_shape=input_shape,
                          pooling=None,
                          type_name='rgb')
    conv_model_rgb = vgg_model_rgb(inputs_rgb)

    ########################

    inputs_depth = layers.Input(shape=input_shape)
    vgg_model_depth = VGG16(include_top=False,
                            weights='vggface',
                            input_tensor=None,
                            input_shape=input_shape,
                            pooling=None,
                            type_name='depth')
    conv_model_depth = vgg_model_depth(inputs_depth)

    ######################
    #    conv_model_depth = vgg_model_depth(inputs_depth)

    #    fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(dropout_rgb)
    #    fc6_depth = layers.Dense(2048, activation='relu', name='fc6_depth')(dropout_depth)

    # CONACTENATE the ends of RGB & DEPTH

    merge_rgb_depth = layers.concatenate([conv_model_rgb, conv_model_depth],
                                         axis=-1)
    attention_features = cbam_block(merge_rgb_depth)
    #

    #
    #    ############ for RGB
    #    flat_model_rgb = layers.Flatten(name='flatten_rgb')(conv_model_rgb)
    #    fc6_rgb = layers.Dense(2048, activation='relu', name='fc6_rgb')(flat_model_rgb)
    #    dropout_rgb = layers.Dropout(0.2)(fc6_rgb)

    ######## for Depth
    flat_model = layers.Flatten(name='flatten')(attention_features)
    fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model)
    bn_1 = BatchNormalization(name='1_bn')(fc6)
    dropout_1 = layers.Dropout(0.5)(bn_1)

    #    flatten_concat = layers.Flatten(name='flatten_concat')(merge_rgb_depth)
    #    fc6 = layers.Dense(2048, activation='relu', name='fc6')(merge_rgb_depth)
    fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1)
    bn_2 = BatchNormalization(name='2_bn')(fc7)
    dropout_2 = layers.Dropout(0.5)(bn_2)

    #    fc8 = layers.Dense(512, activation='relu', name='fc8')(dropout_2)
    #    bn_3 = BatchNormalization(name='3_bn')(fc8)
    #    dropout_3 = layers.Dropout(0.5)(bn_3)

    #VECTORIZING OUTPUT
    output = layers.Dense(n_class, activation='softmax',
                          name='output')(dropout_2)

    # MODAL [INPUTS , OUTPUTS]
    train_model = models.Model(inputs=[inputs_rgb, inputs_depth],
                               outputs=[output])

    #    weights_path = 'CurtinFaces/vgg_multimodal_dropout-0.5_3fc-512/weights-25.h5'
    #    train_model.load_weights(weights_path)
    train_model.summary()
    for layer in train_model.layers[:-23]:
        layer.trainable = False
#    for layer in train_model.layers[2].layers[:-4]:
#        layer.trainable = False
#    for layer in train_model.layers[3].layers[:-4]:
#        layer.trainable = False

    return train_model
コード例 #7
0
def VGGnet(input_shape, n_class):
    """
    
    :param input_shape: data shape, 3d, [width, height, channels]
    :param n_class: number of classes
    
    :return:  Keras Model used for training
    """
#    x = layers.Input(shape=input_shape)

    pickle_in = open("vgg16_face_conv_weights_name.pkl","rb")
    w_and_b = pickle.load(pickle_in)
    add_weights = np.expand_dims(w_and_b['conv1_1'][0][1], axis=2)
#    add_weights = np.zeros((3,3,1,64))  initiate depth channel with zeros
    w_and_b['conv1_1'][0] = np.concatenate((w_and_b['conv1_1'][0],add_weights), axis = 2)
    # DEPTH MODALITY BRANCH OF CNN
    inputs_rgb_d = layers.Input(shape=input_shape)
    #######################VGG/RESNET or any other network
#    vgg_model_depth = VGGFace(include_top=False, input_shape=input_shape)
    # Block 1
    x = layers.Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv1_1'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv1_1'][1]), name='conv1_1d')(inputs_rgb_d)
    x = layers.Conv2D(64, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv1_2'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv1_2'][1]), name='conv1_2d')(x)
    x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool1d')(x)

    # Block 2
    x = layers.Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv2_1'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv2_1'][1]), name='conv2_1d')(
        x)
    x = layers.Conv2D(128, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv2_2'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv2_1'][1]), name='conv2_2d')(
        x)
    x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool2d')(x)

    # Block 3
    x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv3_1'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv3_1'][1]), name='conv3_1d')(
        x)
    x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv3_2'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv3_2'][1]), name='conv3_2d')(
        x)
    x = layers.Conv2D(256, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv3_3'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv3_3'][1]), name='conv3_3d')(
        x)
    x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool3d')(x)

    # Block 4
    x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv4_1'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv4_1'][1]), name='conv4_1d')(
        x)
    x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv4_2'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv4_2'][1]), name='conv4_2d')(
        x)
    x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv4_3'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv4_3'][1]), name='conv4_3d')(
        x)
    x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool4d')(x)

    # Block 5
    x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv5_1'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv5_1'][1]), name='conv5_1d')(
        x)
    x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv5_2'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv5_2'][1]), name='conv5_2d')(
        x)
    x = layers.Conv2D(512, (3, 3), activation='relu', padding='same', kernel_initializer= keras.initializers.Constant(w_and_b['conv5_3'][0]),bias_initializer= keras.initializers.Constant(w_and_b['conv5_3'][1]), name='conv5_3d')(
        x)
    conv_model_rgb = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool5d')(x)
    attention_features = cbam_block(conv_model_rgb, att_type='Channel')
    flat_model = layers.Flatten(name='flatten')(attention_features)
    fc6 = layers.Dense(1024, activation='relu', name='fc6')(flat_model)
    dropout_layer_1 = layers.Dropout(0.2)(fc6)
    fc7 = layers.Dense(512, activation='relu', name='fc7')(dropout_layer_1)
    dropout_layer_2 = layers.Dropout(0.2)(fc7)
    output = layers.Dense(n_class, activation='softmax', name='output')(dropout_layer_2)

#     Models for training and evaluation (prediction)
    train_model = models.Model(inputs_rgb_d,  output)
#    for layer in train_model.layers[:-14]:
#        layer.trainable = False


    return train_model
コード例 #8
0
def VGGFace_multimodal(input_shape, n_class):
    """
    
    :param input_shape: data shape, 3d, [width, height, channels]
    :param n_class: number of classes
    
    :return:  Keras Model used for training
    """
    # RGB MODALITY BRANCH OF CNN
    inputs_rgb = layers.Input(shape=input_shape)
    #    ########################VGG/RESNET or any other network
    resnet_model_rgb = RESNET50(include_top=False,
                                weights='vggface',
                                input_tensor=None,
                                input_shape=None,
                                pooling=None,
                                type_name='rgb')
    #    vgg_model_rgb = VGGFace(include_top=False, input_shape=input_shape)
    conv_model_rgb = resnet_model_rgb(inputs_rgb)
    #
    ######################## DEPTH branch

    inputs_depth = layers.Input(shape=input_shape)
    pre_trained_model = RESNET50(include_top=False,
                                 weights='vggface',
                                 input_tensor=None,
                                 input_shape=None,
                                 pooling=None,
                                 type_name='depth')
    #    pre_trained_model = VGGFace(include_top=False, input_shape=input_shape)
    #    pre_trained_model.summary()
    conv_model_depth = pre_trained_model(inputs_depth)
    #    flat_model_depth = layers.Flatten(name='flatten_depth')(conv_model_depth)
    #
    #
    #    fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model_depth)
    #    dropout_1 = layers.Dropout(0.2)(fc6)
    #
    #    fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1)
    #    dropout_2 = layers.Dropout(0.2)(fc7)
    #
    ##    final_layer = pre_trained_model.layers[-3].output
    #    output = layers.Dense(20, activation='softmax', name='output')(dropout_2)
    #
    #    # Models for training and evaluation (prediction)
    #    train_model = models.Model(inputs_depth,  output)
    #    weights_path =  ('D:/tutorial/rgb+depth+thermal/allData/resne_depth/weights-19.h5')
    #    train_model.load_weights(weights_path)
    #
    #    final_layer = train_model.layers[-7].layers[-1].output
    #    resnet_model_depth = models.Model(inputs_depth,final_layer)
    #
    #
    #    conv_model_depth_1 = resnet_model_depth(inputs_depth)

    merge_rgb_depth = layers.concatenate([conv_model_rgb, conv_model_depth],
                                         axis=-1)
    attention_features = cbam_block(merge_rgb_depth)

    flat_model = layers.Flatten(name='flatten')(attention_features)
    fc6 = layers.Dense(2048, activation='relu', name='fc6')(flat_model)
    dropout_1 = layers.Dropout(0.5)(fc6)

    #    flatten_concat = layers.Flatten(name='flatten_concat')(merge_rgb_depth)
    #    fc6 = layers.Dense(2048, activation='relu', name='fc6')(merge_rgb_depth)
    fc7 = layers.Dense(1024, activation='relu', name='fc7')(dropout_1)
    dropout_2 = layers.Dropout(0.5)(fc7)

    #VECTORIZING OUTPUT
    output = layers.Dense(n_class, activation='softmax',
                          name='output')(dropout_2)

    # MODAL [INPUTS , OUTPUTS]
    train_model = models.Model(inputs=[inputs_rgb, inputs_depth],
                               outputs=[output])

    train_model.summary()
    for layer in train_model.layers[:-19]:
        layer.trainable = False

    return train_model