def valid_entrant(img): model = VGGFace(input_shape=(224, 224, 3)) # (weights='vggface') for default model.load_weights("model_weights.h5") # custom trained im = image.img_to_array(img) im2 = transform.resize(im, (224, 224, 3)) #resize input image preds = model.predict(im2) x = (preds[0][0]) # class label of our most likely prediction if x in permitted: return True else: return False
def custom_vgg_model_w_landmarks(is_trainable, layer_regularization): global sess2 global graph with graph.as_default(): sess2 = tf.compat.v1.Session() graph = tf.compat.v1.get_default_graph() K.set_session(sess2) if layer_regularization == False: model_VGGFace = VGGFace(model='resnet50', include_top=False, input_shape=(224, 224, 3), pooling='avg') # model_VGGFace.trainable = True # for layer in model_VGGFace.layers[:-10]: ## all layers except the last .. layers # layer.trainable = False regularizer = keras.regularizers.l2(0.01) for layer in model_VGGFace.layers: layer.trainable = is_trainable for attr in ['kernel_regularizer']: if hasattr(layer, attr): setattr(layer, attr, regularizer) model_VGGFace.save_weights( "model_checkpoints/VGGFace_Regularization.h5") model_json = model_VGGFace.to_json() model_VGGFace = keras.models.model_from_json(model_json) model_VGGFace.load_weights( "model_checkpoints/VGGFace_Regularization.h5", by_name=True) last_layer = model_VGGFace.get_layer('avg_pool').output x = Flatten(name='flatten')(last_layer) landmarks_input = Input(shape=(136, )) x = Concatenate(axis=-1)([x, landmarks_input]) x = Dense(1024, activation='relu')(x) x = Dropout(0.3)(x) x = BatchNormalization()(x) out1 = Dense(1, activation='tanh', name='out1')(x) out2 = Dense(1, activation='tanh', name='out2')(x) custom_vgg_model = Model( inputs=[model_VGGFace.input, landmarks_input], outputs=[out1, out2]) return custom_vgg_model else: model_VGGFace = VGGFace(model='resnet50', include_top=False, input_shape=(224, 224, 3), pooling='avg') for layer in model_VGGFace.layers: layer.trainable = is_trainable last_layer = model_VGGFace.get_layer('avg_pool').output x = Flatten(name='flatten')(last_layer) landmarks_input = Input(shape=(136, )) x = Concatenate(axis=-1)([x, landmarks_input]) x = Dense(1024, activation='relu')(x) x = Dropout(0.3)(x) x = BatchNormalization()(x) out1 = Dense(1, activation='tanh', name='out1')(x) out2 = Dense(1, activation='tanh', name='out2')(x) custom_vgg_model = Model( inputs=[model_VGGFace.input, landmarks_input], outputs=[out1, out2]) return custom_vgg_model
def custom_vgg_model(is_trainable, conv_block, layer_regularization, model_option): global sess2 global graph with graph.as_default(): K.set_session(sess2) if model_option == -1: # ResNet50 from scracth without separate mask !! model_input = Input(shape=(224, 224, 3)) x = Conv2D(filters=64, kernel_size=(7, 7), strides=2)(model_input) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x1 = MaxPool2D((3, 3), strides=2, padding="same")(x) # Stage 1 # Conv block x = Conv2D(filters=64, kernel_size=(1, 1), strides=1, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=64, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=256, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x1 = Conv2D(filters=256, kernel_size=(1, 1), strides=1, padding='valid')(x1) x1 = BatchNormalization()(x1) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 1 # Identity block for i in [0, 1]: x = Conv2D(filters=64, kernel_size=(1, 1), strides=1, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=64, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=256, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 2 # Conv block x = Conv2D(filters=128, kernel_size=(1, 1), strides=2, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=128, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=512, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x1 = Conv2D(filters=512, kernel_size=(1, 1), strides=2, padding='valid')(x1) x1 = BatchNormalization()(x1) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 2 # Identity block for i in [0, 1, 2]: x = Conv2D(filters=128, kernel_size=(1, 1), strides=1, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=128, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=512, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 3 # Conv block x = Conv2D(filters=256, kernel_size=(1, 1), strides=2, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=256, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=1024, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x1 = Conv2D(filters=1024, kernel_size=(1, 1), strides=2, padding='valid')(x1) x1 = BatchNormalization()(x1) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 3 # Identity block for i in [0, 1, 2, 3, 4]: x = Conv2D(filters=256, kernel_size=(1, 1), strides=1, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=256, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=1024, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 4 # Conv block x = Conv2D(filters=512, kernel_size=(1, 1), strides=2, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=512, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=2048, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x1 = Conv2D(filters=2048, kernel_size=(1, 1), strides=2, padding='valid')(x1) x1 = BatchNormalization()(x1) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 4 # Identity block for i in [0, 1]: x = Conv2D(filters=512, kernel_size=(1, 1), strides=1, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=512, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=2048, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) x = GlobalAveragePooling2D()(x1) x = Dropout(0.7)(x) x = Dense(1024, activation='relu')(x) x = BatchNormalization()(x) x = Dropout(0.6)(x) out1 = Dense(1, activation='tanh', name='out1')(x) out2 = Dense(1, activation='tanh', name='out2')(x) custom_vgg_model = Model(inputs=[model_input], outputs=[out1, out2]) return custom_vgg_model elif model_option == 4: # self coded ResNet50 if RESNET == 18: model_input = Input(shape=(224, 224, 4)) x = Conv2D(filters=64, kernel_size=(7, 7), strides=2)(model_input) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x1 = MaxPool2D((3, 3), strides=2, padding="same")(x) # Stage 1 for i in [0, 1]: x = Conv2D(filters=64, kernel_size=(3, 3), strides=1, padding='same')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=64, kernel_size=(3, 3), strides=1, padding='same')(x) x2 = BatchNormalization()(x) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 2 # Conv block x = Conv2D(filters=128, kernel_size=(3, 3), strides=2, padding='same')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=128, kernel_size=(3, 3), strides=1, padding='same')(x) x2 = BatchNormalization()(x) x1 = Conv2D(filters=128, kernel_size=(1, 1), strides=2, padding='valid')(x1) x1 = BatchNormalization()(x1) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 2 # Identity block x = Conv2D(filters=128, kernel_size=(3, 3), strides=1, padding='same')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=128, kernel_size=(3, 3), strides=1, padding='same')(x) x2 = BatchNormalization()(x) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 3 # Conv block x = Conv2D(filters=256, kernel_size=(3, 3), strides=2, padding='same')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=256, kernel_size=(3, 3), strides=1, padding='same')(x) x2 = BatchNormalization()(x) x1 = Conv2D(filters=256, kernel_size=(1, 1), strides=2, padding='valid')(x1) x1 = BatchNormalization()(x1) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 3 # Identity block x = Conv2D(filters=256, kernel_size=(3, 3), strides=1, padding='same')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=256, kernel_size=(3, 3), strides=1, padding='same')(x) x2 = BatchNormalization()(x) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 4 # Conv block x = Conv2D(filters=512, kernel_size=(3, 3), strides=2, padding='same')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=512, kernel_size=(3, 3), strides=1, padding='same')(x) x2 = BatchNormalization()(x) x1 = Conv2D(filters=512, kernel_size=(1, 1), strides=2, padding='valid')(x1) x1 = BatchNormalization()(x1) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 4 # Identity block x = Conv2D(filters=512, kernel_size=(3, 3), strides=1, padding='same')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=512, kernel_size=(3, 3), strides=1, padding='same')(x) x2 = BatchNormalization()(x) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) x = GlobalAveragePooling2D()(x1) x = Dense(1024, activation='relu')(x) x = BatchNormalization()(x) x = Dropout(0.6)(x) out1 = Dense(1, activation='tanh', name='out1')(x) out2 = Dense(1, activation='tanh', name='out2')(x) custom_vgg_model = Model(inputs=[model_input], outputs=[out1, out2]) return custom_vgg_model ####################################################################################### ####################################################################################### elif RESNET == 50: model_input = Input(shape=(224, 224, 3)) x = Conv2D(filters=64, kernel_size=(7, 7), strides=2)(model_input) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x1 = MaxPool2D((3, 3), strides=2, padding="same")(x) # Stage 1 # Conv block x = Conv2D(filters=64, kernel_size=(1, 1), strides=1, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=64, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=256, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x1 = Conv2D(filters=256, kernel_size=(1, 1), strides=1, padding='valid')(x1) x1 = BatchNormalization()(x1) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 1 # Identity block for i in [0, 1]: x = Conv2D(filters=64, kernel_size=(1, 1), strides=1, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=64, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=256, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 2 # Conv block x = Conv2D(filters=128, kernel_size=(1, 1), strides=2, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=128, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=512, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x1 = Conv2D(filters=512, kernel_size=(1, 1), strides=2, padding='valid')(x1) x1 = BatchNormalization()(x1) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 2 # Identity block for i in [0, 1, 2]: x = Conv2D(filters=128, kernel_size=(1, 1), strides=1, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=128, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=512, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 3 # Conv block x = Conv2D(filters=256, kernel_size=(1, 1), strides=2, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=256, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=1024, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x1 = Conv2D(filters=1024, kernel_size=(1, 1), strides=2, padding='valid')(x1) x1 = BatchNormalization()(x1) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 3 # Identity block for i in [0, 1, 2, 3, 4]: x = Conv2D(filters=256, kernel_size=(1, 1), strides=1, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=256, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=1024, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 4 # Conv block x = Conv2D(filters=512, kernel_size=(1, 1), strides=2, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=512, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=2048, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x1 = Conv2D(filters=2048, kernel_size=(1, 1), strides=2, padding='valid')(x1) x1 = BatchNormalization()(x1) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) # Stage 4 # Identity block for i in [0, 1]: x = Conv2D(filters=512, kernel_size=(1, 1), strides=1, padding='valid')(x1) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=512, kernel_size=(3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation(activations.relu)(x) x = Conv2D(filters=2048, kernel_size=(1, 1), strides=1, padding='valid')(x) x2 = BatchNormalization()(x) x = Add()([x1, x2]) x1 = Activation(activations.relu)(x) x = GlobalAveragePooling2D()(x1) ####### ####### mask_input = Input(shape=(224, 224, 1)) #y = Reshape((224, 224, 1), input_shape=(224,224))(y_input) y = Conv2D(filters=32, kernel_size=(7, 7), strides=2, activation='relu')(mask_input) y = MaxPool2D((2, 2), padding="valid")(y) y = Conv2D(filters=32, kernel_size=(3, 3), strides=1, activation='relu')(y) y = MaxPool2D((2, 2), padding="valid")(y) y = Conv2D(filters=64, kernel_size=(3, 3), strides=2, activation='relu')(y) y = Conv2D(filters=64, kernel_size=(3, 3), strides=1, activation='relu')(y) y = MaxPool2D((2, 2), padding="valid")(y) y = BatchNormalization()(y) y = Flatten()(y) ####### ####### xy = Concatenate(axis=-1)([x, y]) x = Dropout(0.7)(x) x = Dense(1024, activation='relu')(xy) x = BatchNormalization()(x) x = Dropout(0.6)(x) out1 = Dense(1, activation='tanh', name='out1')(x) out2 = Dense(1, activation='tanh', name='out2')(x) custom_vgg_model = Model(inputs=[model_input, mask_input], outputs=[out1, out2]) return custom_vgg_model elif model_option == 3: # with mask - 4 channel input with VGGFace model_VGGFace = VGGFace(model='resnet50', include_top=False, input_shape=(224, 224, 4), pooling='avg') if is_trainable == False: for layer in model_VGGFace.layers: layer.trainable = False else: if conv_block == 1: l_name = 'conv5_3_3x3' elif conv_block == 2: l_name = 'conv5_2_1x1_increase' elif conv_block == 3: l_name = 'conv5_2_1x1_reduce' elif conv_block == 4: l_name = 'conv5_1_3x3' model_VGGFace.trainable = False set_trainable = False for layer in model_VGGFace.layers: if layer.name == l_name: set_trainable = True layer.trainable = set_trainable last_layer = model_VGGFace.get_layer('avg_pool').output print(last_layer.shape) x = Flatten(name='flatten')(last_layer) x = Dropout(0.7)(x) x = Dense(1024, activation='relu')(x) x = Dropout(0.6)(x) x = BatchNormalization()(x) out1 = Dense(1, activation='tanh', name='out1')(x) out2 = Dense(1, activation='tanh', name='out2')(x) custom_vgg_model = Model(inputs=model_VGGFace, outputs=[out1, out2]) return custom_vgg_model elif model_option == 2: # with mask model_VGGFace = VGGFace(model='resnet50', include_top=False, input_shape=(224, 224, 3), pooling='avg') if is_trainable == False: for layer in model_VGGFace.layers: layer.trainable = False else: if conv_block == 1: l_name = 'conv5_3_3x3' elif conv_block == 2: l_name = 'conv5_2_1x1_increase' elif conv_block == 3: l_name = 'conv5_2_1x1_reduce' elif conv_block == 4: l_name = 'conv5_1_3x3' model_VGGFace.trainable = False set_trainable = False for layer in model_VGGFace.layers: if layer.name == l_name: set_trainable = True layer.trainable = set_trainable last_layer = model_VGGFace.get_layer('avg_pool').output print(last_layer.shape) x = Flatten(name='flatten')(last_layer) ####### ####### y_input = Input(shape=(224, 224)) y = Reshape((224, 224, 1), input_shape=(224, 224))(y_input) y = Conv2D(filters=32, kernel_size=(7, 7), strides=2, activation='relu')(y) y = MaxPool2D((2, 2), padding="valid")(y) y = Conv2D(filters=32, kernel_size=(3, 3), strides=1, activation='relu')(y) y = MaxPool2D((2, 2), padding="valid")(y) y = Conv2D(filters=64, kernel_size=(3, 3), strides=2, activation='relu')(y) y = Conv2D(filters=64, kernel_size=(3, 3), strides=1, activation='relu')(y) y = MaxPool2D((2, 2), padding="valid")(y) y = BatchNormalization()(y) y = Flatten()(y) ####### ####### xy = Concatenate(axis=-1)([x, y]) xy = Dropout(0.7)(xy) xy = Dense(1024, activation='relu')(xy) xy = Dropout(0.6)(xy) xy = BatchNormalization()(xy) out1 = Dense(1, activation='tanh', name='out1')(xy) out2 = Dense(1, activation='tanh', name='out2')(xy) custom_vgg_model = Model(inputs=[model_VGGFace.input, y_input], outputs=[out1, out2]) return custom_vgg_model elif model_option == 1: # with LSTM model_VGGFace = VGGFace(model='resnet50', include_top=False, input_shape=(224, 224, 3), pooling='avg') if is_trainable == False: for layer in model_VGGFace.layers: layer.trainable = False else: if conv_block == 1: l_name = 'conv5_3_3x3' elif conv_block == 2: l_name = 'conv5_2_1x1_increase' elif conv_block == 3: l_name = 'conv5_2_1x1_reduce' elif conv_block == 4: l_name = 'conv5_1_3x3' model_VGGFace.trainable = False set_trainable = False for layer in model_VGGFace.layers: if layer.name == l_name: set_trainable = True layer.trainable = set_trainable model = Sequential() model.add(TimeDistributed(model_VGGFace, input_shape=(5, 224, 224, 3))) model.add(TimeDistributed(Flatten(), name='flatten')) model.add( LSTM(256, activation='relu', return_sequences=False, name='lstm')) model.add(Dropout(0.7, name='dropout')) model.add(Dense(1024, activation='relu', name='dense')) model.add(Dropout(0.6, name='dropout2')) model.add(BatchNormalization(name='batchNorm')) model.add(Dense(2, activation='tanh', name='out')) return model elif layer_regularization == False: resnet = True if resnet == True: model_VGGFace = VGGFace(model='resnet50', include_top=False, input_shape=(224, 224, 3), pooling='avg') if is_trainable == False: for layer in model_VGGFace.layers: layer.trainable = False else: MPFT = True if MPFT == True: if conv_block == 1: l_name = 'conv5_3_1x1_increase' elif conv_block == 2: l_name = 'conv5_2_1x1_increase' elif conv_block == 3: l_name = 'conv5_1_1x1_increase' elif conv_block == 4: l_name = 'conv4_3_1x1_increase' model_VGGFace.trainable = False set_trainable = False for layer in model_VGGFace.layers: if layer.name == l_name: set_trainable = True layer.trainable = set_trainable else: for layer in model_VGGFace.layers: layer.trainable = True last_layer = model_VGGFace.get_layer('avg_pool').output print(last_layer.shape) else: model_VGGFace = VGGFace(model='vgg16', include_top=False, input_shape=(224, 224, 3)) if is_trainable == False: for layer in model_VGGFace.layers: layer.trainable = False else: if conv_block == 1: l_name = 'conv5_3' elif conv_block == 2: l_name = 'conv5_2' elif conv_block == 3: l_name = 'conv5_1' elif conv_block == 4: l_name = 'conv4_3' model_VGGFace.trainable = False set_trainable = False for layer in model_VGGFace.layers: if layer.name == l_name: set_trainable = True layer.trainable = set_trainable last_layer = model_VGGFace.get_layer('pool5').output print(last_layer.shape) x = Flatten(name='flatten')(last_layer) x = Dropout(0.7)(x) # x = Dense(1024, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001))(x) x = Dense(1024, activation='relu')(x) x = Dropout(0.6)(x) x = BatchNormalization()(x) out1 = Dense(1, activation='tanh', name='out1')(x) out2 = Dense(1, activation='tanh', name='out2')(x) custom_vgg_model = Model(inputs=model_VGGFace.input, outputs=[out1, out2]) return custom_vgg_model else: model_VGGFace = VGGFace(model='resnet50', include_top=False, input_shape=(224, 224, 3), pooling='avg') regularizer = keras.regularizers.l2(0.001) if is_trainable == False: for layer in model_VGGFace.layers: layer.trainable = False else: if conv_block == 1: l_name = 'conv5_3_3x3' elif conv_block == 2: l_name = 'conv5_2_1x1_increase' elif conv_block == 3: l_name = 'conv5_2_1x1_reduce' elif conv_block == 4: l_name = 'conv5_1_3x3' model_VGGFace.trainable = False set_trainable = False for layer in model_VGGFace.layers: if layer.name == l_name: set_trainable = True layer.trainable = set_trainable for attr in ['kernel_regularizer']: if hasattr(layer, attr): setattr(layer, attr, regularizer) model_VGGFace.save_weights( "model_checkpoints/VGGFace_Regularization.h5") model_json = model_VGGFace.to_json() model_VGGFace = keras.models.model_from_json(model_json) model_VGGFace.load_weights( "model_checkpoints/VGGFace_Regularization.h5", by_name=True) last_layer = model_VGGFace.get_layer('avg_pool').output x = Flatten(name='flatten')(last_layer) x = Dropout(0.7)(x) x = Dense(1024, activation='relu')(x) x = Dropout(0.6)(x) x = BatchNormalization()(x) out1 = Dense(1, activation='tanh', name='out1')(x) out2 = Dense(1, activation='tanh', name='out2')(x) custom_vgg_model = Model(inputs=model_VGGFace.input, outputs=[out1, out2]) return custom_vgg_model
''' 实现自己的网络机构 ''' #接下来就是考虑什么模型可以来做这个人脸预测了,测试vgg_face,senet50 from keras_vggface.vggface import VGGFace from keras.layers import * from lin import categorical_focal_loss vgg_model = VGGFace(include_top=False, model='senet50', input_shape=(224, 224, 3), weights=None) # 必须使用该方法下载模型,然后加载 from flyai.utils import remote_helper path = remote_helper.get_remote_date( 'https://www.flyai.com/m/rcmalli_vggface_tf_notop_senet50.h5') vgg_model.load_weights(path) x = vgg_model.output x = Flatten()(x) x = BatchNormalization()(x) x = Dense(128, activation='relu')(x) x = Dropout(rate=0.5)(x) x = Dense(128, activation='relu')(x) x = Dropout(rate=0.5)(x) o = Dense(10, activation='softmax', name='prediction')(x) seque = Keras_Model(inputs=vgg_model.input, outputs=o) #学习率设置 from keras.optimizers import * from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint sgd = SGD(lr=0.0001) seque.compile(loss=categorical_focal_loss(gamma=2., alpha=0.5),
def run(img_dirA, img_dirB, TOTAL_ITERS = 40000): global model, vggface img_dirA_bm_eyes = f"{img_dirA}/binary_masks_eyes2" img_dirB_bm_eyes = f"{img_dirB}/binary_masks_eyes2" # Path to saved model weights models_dir = "./models" # Number of CPU cores num_cpus = os.cpu_count() # Input/Output resolution RESOLUTION = 64 # 64x64, 128x128, 256x256 assert (RESOLUTION % 64) == 0, "RESOLUTION should be 64, 128, or 256." # Batch size batchSize = 8 assert (batchSize != 1 and batchSize % 2 == 0) , "batchSize should be an even number." # Use motion blurs (data augmentation) # set True if training data contains images extracted from videos use_da_motion_blur = False # Use eye-aware training # require images generated from prep_binary_masks.ipynb use_bm_eyes = True # Probability of random color matching (data augmentation) prob_random_color_match = 0.5 da_config = { "prob_random_color_match": prob_random_color_match, "use_da_motion_blur": use_da_motion_blur, "use_bm_eyes": use_bm_eyes } # Architecture configuration arch_config = {} arch_config['IMAGE_SHAPE'] = (RESOLUTION, RESOLUTION, 3) arch_config['use_self_attn'] = True arch_config['norm'] = "instancenorm" # instancenorm, batchnorm, layernorm, groupnorm, none arch_config['model_capacity'] = "standard" # standard, lite # VGGFace ResNet50 vggface = VGGFace(include_top=False, model='resnet50', input_shape=(224, 224, 3)) # Loss function weights configuration loss_weights = {} loss_weights['w_D'] = 0.1 # Discriminator loss_weights['w_recon'] = 1. # L1 reconstruction loss loss_weights['w_edge'] = 0.1 # edge loss loss_weights['w_eyes'] = 30. # reconstruction and edge loss on eyes area loss_weights['w_pl'] = (0.01, 0.1, 0.3, 0.1) # perceptual loss (0.003, 0.03, 0.3, 0.3) # Init. loss config. loss_config = {} loss_config["gan_training"] = "mixup_LSGAN" # "mixup_LSGAN" or "relativistic_avg_LSGAN" loss_config['use_PL'] = False loss_config["PL_before_activ"] = False loss_config['use_mask_hinge_loss'] = False loss_config['m_mask'] = 0. loss_config['lr_factor'] = 1. loss_config['use_cyclic_loss'] = False from networks.faceswap_gan_model import FaceswapGANModel model = FaceswapGANModel(**arch_config) model.load_weights(path=models_dir) from colab_demo.vggface_models import RESNET50 vggface = RESNET50(include_top=False, weights=None, input_shape=(224, 224, 3)) vggface.load_weights("rcmalli_vggface_tf_notop_resnet50.h5") #vggface.summary() model.build_pl_model(vggface_model=vggface, before_activ=loss_config["PL_before_activ"]) model.build_train_functions(loss_weights=loss_weights, **loss_config) from data_loader.data_loader import DataLoader # Create ./models directory Path(f"models").mkdir(parents=True, exist_ok=True) # Get filenames faces_A = f"{img_dirA}/raw_faces" faces_B = f"{img_dirB}/raw_faces" train_A = glob.glob(f"{faces_A}/*.*") train_B = glob.glob(f"{faces_B}/*.*") train_AnB = train_A + train_B assert len(train_A), f"No image found in {faces_A}" assert len(train_B), f"No image found in {faces_B}" print ("Number of images in folder A: " + str(len(train_A))) print ("Number of images in folder B: " + str(len(train_B))) if use_bm_eyes: assert len(glob.glob(img_dirA_bm_eyes+"/*.*")), "No binary mask found in " + str(img_dirA_bm_eyes) assert len(glob.glob(img_dirB_bm_eyes+"/*.*")), "No binary mask found in " + str(img_dirB_bm_eyes) assert len(glob.glob(img_dirA_bm_eyes+"/*.*")) == len(train_A), \ "Number of faceA images does not match number of their binary masks. Can be caused by any none image file in the folder." assert len(glob.glob(img_dirB_bm_eyes+"/*.*")) == len(train_B), \ "Number of faceB images does not match number of their binary masks. Can be caused by any none image file in the folder." def show_loss_config(loss_config): for config, value in loss_config.items(): print(f"{config} = {value}") def reset_session(save_path): global model, vggface global train_batchA, train_batchB model.save_weights(path=save_path) del model del vggface del train_batchA del train_batchB K.clear_session() model = FaceswapGANModel(**arch_config) model.load_weights(path=save_path) vggface = VGGFace(include_top=False, model='resnet50', input_shape=(224, 224, 3)) model.build_pl_model(vggface_model=vggface, before_activ=loss_config["PL_before_activ"]) train_batchA = DataLoader(train_A, train_AnB, batchSize, img_dirA_bm_eyes, RESOLUTION, num_cpus, K.get_session(), **da_config) train_batchB = DataLoader(train_B, train_AnB, batchSize, img_dirB_bm_eyes, RESOLUTION, num_cpus, K.get_session(), **da_config) # Start training t0 = time.time() # This try/except is meant to resume training that was accidentally interrupted try: gen_iterations print(f"Resume training from iter {gen_iterations}.") except: gen_iterations = 0 errGA_sum = errGB_sum = errDA_sum = errDB_sum = 0 errGAs = {} errGBs = {} # Dictionaries are ordered in Python 3.6 for k in ['ttl', 'adv', 'recon', 'edge', 'pl']: errGAs[k] = 0 errGBs[k] = 0 display_iters = 300 backup_iters = 5000 global train_batchA, train_batchB train_batchA = DataLoader(train_A, train_AnB, batchSize, img_dirA_bm_eyes, RESOLUTION, num_cpus, K.get_session(), **da_config) train_batchB = DataLoader(train_B, train_AnB, batchSize, img_dirB_bm_eyes, RESOLUTION, num_cpus, K.get_session(), **da_config) clear_output() loss_config['use_PL'] = True loss_config['use_mask_hinge_loss'] = False loss_config['m_mask'] = 0.0 reset_session(models_dir) print("Building new loss funcitons...") show_loss_config(loss_config) model.build_train_functions(loss_weights=loss_weights, **loss_config) print("Done.") while gen_iterations <= TOTAL_ITERS: # Loss function automation if gen_iterations == (TOTAL_ITERS//5 - display_iters//2): clear_output() loss_config['use_PL'] = True loss_config['use_mask_hinge_loss'] = False loss_config['m_mask'] = 0.0 reset_session(models_dir) print("Building new loss funcitons...") show_loss_config(loss_config) model.build_train_functions(loss_weights=loss_weights, **loss_config) print("Done.") elif gen_iterations == (TOTAL_ITERS//5 + TOTAL_ITERS//10 - display_iters//2): clear_output() loss_config['use_PL'] = True loss_config['use_mask_hinge_loss'] = True loss_config['m_mask'] = 0.5 reset_session(models_dir) print("Building new loss funcitons...") show_loss_config(loss_config) model.build_train_functions(loss_weights=loss_weights, **loss_config) print("Complete.") elif gen_iterations == (2*TOTAL_ITERS//5 - display_iters//2): clear_output() loss_config['use_PL'] = True loss_config['use_mask_hinge_loss'] = True loss_config['m_mask'] = 0.2 reset_session(models_dir) print("Building new loss funcitons...") show_loss_config(loss_config) model.build_train_functions(loss_weights=loss_weights, **loss_config) print("Done.") elif gen_iterations == (TOTAL_ITERS//2 - display_iters//2): clear_output() loss_config['use_PL'] = True loss_config['use_mask_hinge_loss'] = True loss_config['m_mask'] = 0.4 reset_session(models_dir) print("Building new loss funcitons...") show_loss_config(loss_config) model.build_train_functions(loss_weights=loss_weights, **loss_config) print("Done.") elif gen_iterations == (2*TOTAL_ITERS//3 - display_iters//2): clear_output() loss_config['use_PL'] = True loss_config['use_mask_hinge_loss'] = False loss_config['m_mask'] = 0. loss_config['lr_factor'] = 0.3 reset_session(models_dir) print("Building new loss funcitons...") show_loss_config(loss_config) model.build_train_functions(loss_weights=loss_weights, **loss_config) print("Done.") elif gen_iterations == (8*TOTAL_ITERS//10 - display_iters//2): clear_output() model.decoder_A.load_weights("models/decoder_B.h5") # swap decoders model.decoder_B.load_weights("models/decoder_A.h5") # swap decoders loss_config['use_PL'] = True loss_config['use_mask_hinge_loss'] = True loss_config['m_mask'] = 0.1 loss_config['lr_factor'] = 0.3 reset_session(models_dir) print("Building new loss funcitons...") show_loss_config(loss_config) model.build_train_functions(loss_weights=loss_weights, **loss_config) print("Done.") elif gen_iterations == (9*TOTAL_ITERS//10 - display_iters//2): clear_output() loss_config['use_PL'] = True loss_config['use_mask_hinge_loss'] = False loss_config['m_mask'] = 0.0 loss_config['lr_factor'] = 0.1 reset_session(models_dir) print("Building new loss funcitons...") show_loss_config(loss_config) model.build_train_functions(loss_weights=loss_weights, **loss_config) print("Done.") if gen_iterations == 5: print ("working.") # Train dicriminators for one batch data_A = train_batchA.get_next_batch() data_B = train_batchB.get_next_batch() errDA, errDB = model.train_one_batch_D(data_A=data_A, data_B=data_B) errDA_sum +=errDA[0] errDB_sum +=errDB[0] # Train generators for one batch data_A = train_batchA.get_next_batch() data_B = train_batchB.get_next_batch() errGA, errGB = model.train_one_batch_G(data_A=data_A, data_B=data_B) errGA_sum += errGA[0] errGB_sum += errGB[0] for i, k in enumerate(['ttl', 'adv', 'recon', 'edge', 'pl']): errGAs[k] += errGA[i] errGBs[k] += errGB[i] gen_iterations+=1 # Visualization if gen_iterations % display_iters == 0: clear_output() # Display loss information show_loss_config(loss_config) print("----------") print('[iter %d] Loss_DA: %f Loss_DB: %f Loss_GA: %f Loss_GB: %f time: %f' % (gen_iterations, errDA_sum/display_iters, errDB_sum/display_iters, errGA_sum/display_iters, errGB_sum/display_iters, time.time()-t0)) print("----------") print("Generator loss details:") print(f'[Adversarial loss]') print(f'GA: {errGAs["adv"]/display_iters:.4f} GB: {errGBs["adv"]/display_iters:.4f}') print(f'[Reconstruction loss]') print(f'GA: {errGAs["recon"]/display_iters:.4f} GB: {errGBs["recon"]/display_iters:.4f}') print(f'[Edge loss]') print(f'GA: {errGAs["edge"]/display_iters:.4f} GB: {errGBs["edge"]/display_iters:.4f}') if loss_config['use_PL'] == True: print(f'[Perceptual loss]') try: print(f'GA: {errGAs["pl"][0]/display_iters:.4f} GB: {errGBs["pl"][0]/display_iters:.4f}') except: print(f'GA: {errGAs["pl"]/display_iters:.4f} GB: {errGBs["pl"]/display_iters:.4f}') # Display images print("----------") wA, tA, _ = train_batchA.get_next_batch() wB, tB, _ = train_batchB.get_next_batch() print("Transformed (masked) results:") showG(tA, tB, model.path_A, model.path_B, batchSize) print("Masks:") showG_mask(tA, tB, model.path_mask_A, model.path_mask_B, batchSize) print("Reconstruction results:") showG(wA, wB, model.path_bgr_A, model.path_bgr_B, batchSize) errGA_sum = errGB_sum = errDA_sum = errDB_sum = 0 for k in ['ttl', 'adv', 'recon', 'edge', 'pl']: errGAs[k] = 0 errGBs[k] = 0 # Save models model.save_weights(path=models_dir) # Backup models if gen_iterations % backup_iters == 0: bkup_dir = f"{models_dir}/backup_iter{gen_iterations}" Path(bkup_dir).mkdir(parents=True, exist_ok=True) model.save_weights(path=bkup_dir)