def valid_entrant(img):
model = VGGFace(input_shape=(224, 224,
3)) # (weights='vggface') for default
model.load_weights("model_weights.h5") # custom trained
im = image.img_to_array(img)
im2 = transform.resize(im, (224, 224, 3)) #resize input image
preds = model.predict(im2)
x = (preds[0][0]) # class label of our most likely prediction
if x in permitted:
return True
else:
return False
def custom_vgg_model_w_landmarks(is_trainable, layer_regularization):
global sess2
global graph
with graph.as_default():
sess2 = tf.compat.v1.Session()
graph = tf.compat.v1.get_default_graph()
K.set_session(sess2)
if layer_regularization == False:
model_VGGFace = VGGFace(model='resnet50',
include_top=False,
input_shape=(224, 224, 3),
pooling='avg')
# model_VGGFace.trainable = True
# for layer in model_VGGFace.layers[:-10]: ## all layers except the last .. layers
# layer.trainable = False
regularizer = keras.regularizers.l2(0.01)
for layer in model_VGGFace.layers:
layer.trainable = is_trainable
for attr in ['kernel_regularizer']:
if hasattr(layer, attr):
setattr(layer, attr, regularizer)
model_VGGFace.save_weights(
"model_checkpoints/VGGFace_Regularization.h5")
model_json = model_VGGFace.to_json()
model_VGGFace = keras.models.model_from_json(model_json)
model_VGGFace.load_weights(
"model_checkpoints/VGGFace_Regularization.h5", by_name=True)
last_layer = model_VGGFace.get_layer('avg_pool').output
x = Flatten(name='flatten')(last_layer)
landmarks_input = Input(shape=(136, ))
x = Concatenate(axis=-1)([x, landmarks_input])
x = Dense(1024, activation='relu')(x)
x = Dropout(0.3)(x)
x = BatchNormalization()(x)
out1 = Dense(1, activation='tanh', name='out1')(x)
out2 = Dense(1, activation='tanh', name='out2')(x)
custom_vgg_model = Model(
inputs=[model_VGGFace.input, landmarks_input],
outputs=[out1, out2])
return custom_vgg_model
else:
model_VGGFace = VGGFace(model='resnet50',
include_top=False,
input_shape=(224, 224, 3),
pooling='avg')
for layer in model_VGGFace.layers:
layer.trainable = is_trainable
last_layer = model_VGGFace.get_layer('avg_pool').output
x = Flatten(name='flatten')(last_layer)
landmarks_input = Input(shape=(136, ))
x = Concatenate(axis=-1)([x, landmarks_input])
x = Dense(1024, activation='relu')(x)
x = Dropout(0.3)(x)
x = BatchNormalization()(x)
out1 = Dense(1, activation='tanh', name='out1')(x)
out2 = Dense(1, activation='tanh', name='out2')(x)
custom_vgg_model = Model(
inputs=[model_VGGFace.input, landmarks_input],
outputs=[out1, out2])
return custom_vgg_model
def custom_vgg_model(is_trainable, conv_block, layer_regularization,
model_option):
global sess2
global graph
with graph.as_default():
K.set_session(sess2)
if model_option == -1: # ResNet50 from scracth without separate mask !!
model_input = Input(shape=(224, 224, 3))
x = Conv2D(filters=64, kernel_size=(7, 7), strides=2)(model_input)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x1 = MaxPool2D((3, 3), strides=2, padding="same")(x)
# Stage 1
# Conv block
x = Conv2D(filters=64, kernel_size=(1, 1), strides=1,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=64, kernel_size=(3, 3), strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=256, kernel_size=(1, 1), strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x1 = Conv2D(filters=256,
kernel_size=(1, 1),
strides=1,
padding='valid')(x1)
x1 = BatchNormalization()(x1)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 1
# Identity block
for i in [0, 1]:
x = Conv2D(filters=64,
kernel_size=(1, 1),
strides=1,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=64,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=256,
kernel_size=(1, 1),
strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 2
# Conv block
x = Conv2D(filters=128, kernel_size=(1, 1), strides=2,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=128, kernel_size=(3, 3), strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=512, kernel_size=(1, 1), strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x1 = Conv2D(filters=512,
kernel_size=(1, 1),
strides=2,
padding='valid')(x1)
x1 = BatchNormalization()(x1)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 2
# Identity block
for i in [0, 1, 2]:
x = Conv2D(filters=128,
kernel_size=(1, 1),
strides=1,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=128,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=512,
kernel_size=(1, 1),
strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 3
# Conv block
x = Conv2D(filters=256, kernel_size=(1, 1), strides=2,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=256, kernel_size=(3, 3), strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=1024,
kernel_size=(1, 1),
strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x1 = Conv2D(filters=1024,
kernel_size=(1, 1),
strides=2,
padding='valid')(x1)
x1 = BatchNormalization()(x1)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 3
# Identity block
for i in [0, 1, 2, 3, 4]:
x = Conv2D(filters=256,
kernel_size=(1, 1),
strides=1,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=256,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=1024,
kernel_size=(1, 1),
strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 4
# Conv block
x = Conv2D(filters=512, kernel_size=(1, 1), strides=2,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=512, kernel_size=(3, 3), strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=2048,
kernel_size=(1, 1),
strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x1 = Conv2D(filters=2048,
kernel_size=(1, 1),
strides=2,
padding='valid')(x1)
x1 = BatchNormalization()(x1)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 4
# Identity block
for i in [0, 1]:
x = Conv2D(filters=512,
kernel_size=(1, 1),
strides=1,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=512,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=2048,
kernel_size=(1, 1),
strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
x = GlobalAveragePooling2D()(x1)
x = Dropout(0.7)(x)
x = Dense(1024, activation='relu')(x)
x = BatchNormalization()(x)
x = Dropout(0.6)(x)
out1 = Dense(1, activation='tanh', name='out1')(x)
out2 = Dense(1, activation='tanh', name='out2')(x)
custom_vgg_model = Model(inputs=[model_input], outputs=[out1, out2])
return custom_vgg_model
elif model_option == 4: # self coded ResNet50
if RESNET == 18:
model_input = Input(shape=(224, 224, 4))
x = Conv2D(filters=64, kernel_size=(7, 7), strides=2)(model_input)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x1 = MaxPool2D((3, 3), strides=2, padding="same")(x)
# Stage 1
for i in [0, 1]:
x = Conv2D(filters=64,
kernel_size=(3, 3),
strides=1,
padding='same')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=64,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x2 = BatchNormalization()(x)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 2
# Conv block
x = Conv2D(filters=128,
kernel_size=(3, 3),
strides=2,
padding='same')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=128,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x2 = BatchNormalization()(x)
x1 = Conv2D(filters=128,
kernel_size=(1, 1),
strides=2,
padding='valid')(x1)
x1 = BatchNormalization()(x1)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 2
# Identity block
x = Conv2D(filters=128,
kernel_size=(3, 3),
strides=1,
padding='same')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=128,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x2 = BatchNormalization()(x)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 3
# Conv block
x = Conv2D(filters=256,
kernel_size=(3, 3),
strides=2,
padding='same')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=256,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x2 = BatchNormalization()(x)
x1 = Conv2D(filters=256,
kernel_size=(1, 1),
strides=2,
padding='valid')(x1)
x1 = BatchNormalization()(x1)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 3
# Identity block
x = Conv2D(filters=256,
kernel_size=(3, 3),
strides=1,
padding='same')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=256,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x2 = BatchNormalization()(x)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 4
# Conv block
x = Conv2D(filters=512,
kernel_size=(3, 3),
strides=2,
padding='same')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=512,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x2 = BatchNormalization()(x)
x1 = Conv2D(filters=512,
kernel_size=(1, 1),
strides=2,
padding='valid')(x1)
x1 = BatchNormalization()(x1)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 4
# Identity block
x = Conv2D(filters=512,
kernel_size=(3, 3),
strides=1,
padding='same')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=512,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x2 = BatchNormalization()(x)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
x = GlobalAveragePooling2D()(x1)
x = Dense(1024, activation='relu')(x)
x = BatchNormalization()(x)
x = Dropout(0.6)(x)
out1 = Dense(1, activation='tanh', name='out1')(x)
out2 = Dense(1, activation='tanh', name='out2')(x)
custom_vgg_model = Model(inputs=[model_input],
outputs=[out1, out2])
return custom_vgg_model
#######################################################################################
#######################################################################################
elif RESNET == 50:
model_input = Input(shape=(224, 224, 3))
x = Conv2D(filters=64, kernel_size=(7, 7), strides=2)(model_input)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x1 = MaxPool2D((3, 3), strides=2, padding="same")(x)
# Stage 1
# Conv block
x = Conv2D(filters=64,
kernel_size=(1, 1),
strides=1,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=64,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=256,
kernel_size=(1, 1),
strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x1 = Conv2D(filters=256,
kernel_size=(1, 1),
strides=1,
padding='valid')(x1)
x1 = BatchNormalization()(x1)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 1
# Identity block
for i in [0, 1]:
x = Conv2D(filters=64,
kernel_size=(1, 1),
strides=1,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=64,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=256,
kernel_size=(1, 1),
strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 2
# Conv block
x = Conv2D(filters=128,
kernel_size=(1, 1),
strides=2,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=128,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=512,
kernel_size=(1, 1),
strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x1 = Conv2D(filters=512,
kernel_size=(1, 1),
strides=2,
padding='valid')(x1)
x1 = BatchNormalization()(x1)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 2
# Identity block
for i in [0, 1, 2]:
x = Conv2D(filters=128,
kernel_size=(1, 1),
strides=1,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=128,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=512,
kernel_size=(1, 1),
strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 3
# Conv block
x = Conv2D(filters=256,
kernel_size=(1, 1),
strides=2,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=256,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=1024,
kernel_size=(1, 1),
strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x1 = Conv2D(filters=1024,
kernel_size=(1, 1),
strides=2,
padding='valid')(x1)
x1 = BatchNormalization()(x1)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 3
# Identity block
for i in [0, 1, 2, 3, 4]:
x = Conv2D(filters=256,
kernel_size=(1, 1),
strides=1,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=256,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=1024,
kernel_size=(1, 1),
strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 4
# Conv block
x = Conv2D(filters=512,
kernel_size=(1, 1),
strides=2,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=512,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=2048,
kernel_size=(1, 1),
strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x1 = Conv2D(filters=2048,
kernel_size=(1, 1),
strides=2,
padding='valid')(x1)
x1 = BatchNormalization()(x1)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
# Stage 4
# Identity block
for i in [0, 1]:
x = Conv2D(filters=512,
kernel_size=(1, 1),
strides=1,
padding='valid')(x1)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=512,
kernel_size=(3, 3),
strides=1,
padding='same')(x)
x = BatchNormalization()(x)
x = Activation(activations.relu)(x)
x = Conv2D(filters=2048,
kernel_size=(1, 1),
strides=1,
padding='valid')(x)
x2 = BatchNormalization()(x)
x = Add()([x1, x2])
x1 = Activation(activations.relu)(x)
x = GlobalAveragePooling2D()(x1)
#######
#######
mask_input = Input(shape=(224, 224, 1))
#y = Reshape((224, 224, 1), input_shape=(224,224))(y_input)
y = Conv2D(filters=32,
kernel_size=(7, 7),
strides=2,
activation='relu')(mask_input)
y = MaxPool2D((2, 2), padding="valid")(y)
y = Conv2D(filters=32,
kernel_size=(3, 3),
strides=1,
activation='relu')(y)
y = MaxPool2D((2, 2), padding="valid")(y)
y = Conv2D(filters=64,
kernel_size=(3, 3),
strides=2,
activation='relu')(y)
y = Conv2D(filters=64,
kernel_size=(3, 3),
strides=1,
activation='relu')(y)
y = MaxPool2D((2, 2), padding="valid")(y)
y = BatchNormalization()(y)
y = Flatten()(y)
#######
#######
xy = Concatenate(axis=-1)([x, y])
x = Dropout(0.7)(x)
x = Dense(1024, activation='relu')(xy)
x = BatchNormalization()(x)
x = Dropout(0.6)(x)
out1 = Dense(1, activation='tanh', name='out1')(x)
out2 = Dense(1, activation='tanh', name='out2')(x)
custom_vgg_model = Model(inputs=[model_input, mask_input],
outputs=[out1, out2])
return custom_vgg_model
elif model_option == 3: # with mask - 4 channel input with VGGFace
model_VGGFace = VGGFace(model='resnet50',
include_top=False,
input_shape=(224, 224, 4),
pooling='avg')
if is_trainable == False:
for layer in model_VGGFace.layers:
layer.trainable = False
else:
if conv_block == 1:
l_name = 'conv5_3_3x3'
elif conv_block == 2:
l_name = 'conv5_2_1x1_increase'
elif conv_block == 3:
l_name = 'conv5_2_1x1_reduce'
elif conv_block == 4:
l_name = 'conv5_1_3x3'
model_VGGFace.trainable = False
set_trainable = False
for layer in model_VGGFace.layers:
if layer.name == l_name:
set_trainable = True
layer.trainable = set_trainable
last_layer = model_VGGFace.get_layer('avg_pool').output
print(last_layer.shape)
x = Flatten(name='flatten')(last_layer)
x = Dropout(0.7)(x)
x = Dense(1024, activation='relu')(x)
x = Dropout(0.6)(x)
x = BatchNormalization()(x)
out1 = Dense(1, activation='tanh', name='out1')(x)
out2 = Dense(1, activation='tanh', name='out2')(x)
custom_vgg_model = Model(inputs=model_VGGFace, outputs=[out1, out2])
return custom_vgg_model
elif model_option == 2: # with mask
model_VGGFace = VGGFace(model='resnet50',
include_top=False,
input_shape=(224, 224, 3),
pooling='avg')
if is_trainable == False:
for layer in model_VGGFace.layers:
layer.trainable = False
else:
if conv_block == 1:
l_name = 'conv5_3_3x3'
elif conv_block == 2:
l_name = 'conv5_2_1x1_increase'
elif conv_block == 3:
l_name = 'conv5_2_1x1_reduce'
elif conv_block == 4:
l_name = 'conv5_1_3x3'
model_VGGFace.trainable = False
set_trainable = False
for layer in model_VGGFace.layers:
if layer.name == l_name:
set_trainable = True
layer.trainable = set_trainable
last_layer = model_VGGFace.get_layer('avg_pool').output
print(last_layer.shape)
x = Flatten(name='flatten')(last_layer)
#######
#######
y_input = Input(shape=(224, 224))
y = Reshape((224, 224, 1), input_shape=(224, 224))(y_input)
y = Conv2D(filters=32,
kernel_size=(7, 7),
strides=2,
activation='relu')(y)
y = MaxPool2D((2, 2), padding="valid")(y)
y = Conv2D(filters=32,
kernel_size=(3, 3),
strides=1,
activation='relu')(y)
y = MaxPool2D((2, 2), padding="valid")(y)
y = Conv2D(filters=64,
kernel_size=(3, 3),
strides=2,
activation='relu')(y)
y = Conv2D(filters=64,
kernel_size=(3, 3),
strides=1,
activation='relu')(y)
y = MaxPool2D((2, 2), padding="valid")(y)
y = BatchNormalization()(y)
y = Flatten()(y)
#######
#######
xy = Concatenate(axis=-1)([x, y])
xy = Dropout(0.7)(xy)
xy = Dense(1024, activation='relu')(xy)
xy = Dropout(0.6)(xy)
xy = BatchNormalization()(xy)
out1 = Dense(1, activation='tanh', name='out1')(xy)
out2 = Dense(1, activation='tanh', name='out2')(xy)
custom_vgg_model = Model(inputs=[model_VGGFace.input, y_input],
outputs=[out1, out2])
return custom_vgg_model
elif model_option == 1: # with LSTM
model_VGGFace = VGGFace(model='resnet50',
include_top=False,
input_shape=(224, 224, 3),
pooling='avg')
if is_trainable == False:
for layer in model_VGGFace.layers:
layer.trainable = False
else:
if conv_block == 1:
l_name = 'conv5_3_3x3'
elif conv_block == 2:
l_name = 'conv5_2_1x1_increase'
elif conv_block == 3:
l_name = 'conv5_2_1x1_reduce'
elif conv_block == 4:
l_name = 'conv5_1_3x3'
model_VGGFace.trainable = False
set_trainable = False
for layer in model_VGGFace.layers:
if layer.name == l_name:
set_trainable = True
layer.trainable = set_trainable
model = Sequential()
model.add(TimeDistributed(model_VGGFace, input_shape=(5, 224, 224, 3)))
model.add(TimeDistributed(Flatten(), name='flatten'))
model.add(
LSTM(256, activation='relu', return_sequences=False, name='lstm'))
model.add(Dropout(0.7, name='dropout'))
model.add(Dense(1024, activation='relu', name='dense'))
model.add(Dropout(0.6, name='dropout2'))
model.add(BatchNormalization(name='batchNorm'))
model.add(Dense(2, activation='tanh', name='out'))
return model
elif layer_regularization == False:
resnet = True
if resnet == True:
model_VGGFace = VGGFace(model='resnet50',
include_top=False,
input_shape=(224, 224, 3),
pooling='avg')
if is_trainable == False:
for layer in model_VGGFace.layers:
layer.trainable = False
else:
MPFT = True
if MPFT == True:
if conv_block == 1:
l_name = 'conv5_3_1x1_increase'
elif conv_block == 2:
l_name = 'conv5_2_1x1_increase'
elif conv_block == 3:
l_name = 'conv5_1_1x1_increase'
elif conv_block == 4:
l_name = 'conv4_3_1x1_increase'
model_VGGFace.trainable = False
set_trainable = False
for layer in model_VGGFace.layers:
if layer.name == l_name:
set_trainable = True
layer.trainable = set_trainable
else:
for layer in model_VGGFace.layers:
layer.trainable = True
last_layer = model_VGGFace.get_layer('avg_pool').output
print(last_layer.shape)
else:
model_VGGFace = VGGFace(model='vgg16',
include_top=False,
input_shape=(224, 224, 3))
if is_trainable == False:
for layer in model_VGGFace.layers:
layer.trainable = False
else:
if conv_block == 1:
l_name = 'conv5_3'
elif conv_block == 2:
l_name = 'conv5_2'
elif conv_block == 3:
l_name = 'conv5_1'
elif conv_block == 4:
l_name = 'conv4_3'
model_VGGFace.trainable = False
set_trainable = False
for layer in model_VGGFace.layers:
if layer.name == l_name:
set_trainable = True
layer.trainable = set_trainable
last_layer = model_VGGFace.get_layer('pool5').output
print(last_layer.shape)
x = Flatten(name='flatten')(last_layer)
x = Dropout(0.7)(x)
# x = Dense(1024, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001))(x)
x = Dense(1024, activation='relu')(x)
x = Dropout(0.6)(x)
x = BatchNormalization()(x)
out1 = Dense(1, activation='tanh', name='out1')(x)
out2 = Dense(1, activation='tanh', name='out2')(x)
custom_vgg_model = Model(inputs=model_VGGFace.input,
outputs=[out1, out2])
return custom_vgg_model
else:
model_VGGFace = VGGFace(model='resnet50',
include_top=False,
input_shape=(224, 224, 3),
pooling='avg')
regularizer = keras.regularizers.l2(0.001)
if is_trainable == False:
for layer in model_VGGFace.layers:
layer.trainable = False
else:
if conv_block == 1:
l_name = 'conv5_3_3x3'
elif conv_block == 2:
l_name = 'conv5_2_1x1_increase'
elif conv_block == 3:
l_name = 'conv5_2_1x1_reduce'
elif conv_block == 4:
l_name = 'conv5_1_3x3'
model_VGGFace.trainable = False
set_trainable = False
for layer in model_VGGFace.layers:
if layer.name == l_name:
set_trainable = True
layer.trainable = set_trainable
for attr in ['kernel_regularizer']:
if hasattr(layer, attr):
setattr(layer, attr, regularizer)
model_VGGFace.save_weights(
"model_checkpoints/VGGFace_Regularization.h5")
model_json = model_VGGFace.to_json()
model_VGGFace = keras.models.model_from_json(model_json)
model_VGGFace.load_weights(
"model_checkpoints/VGGFace_Regularization.h5", by_name=True)
last_layer = model_VGGFace.get_layer('avg_pool').output
x = Flatten(name='flatten')(last_layer)
x = Dropout(0.7)(x)
x = Dense(1024, activation='relu')(x)
x = Dropout(0.6)(x)
x = BatchNormalization()(x)
out1 = Dense(1, activation='tanh', name='out1')(x)
out2 = Dense(1, activation='tanh', name='out2')(x)
custom_vgg_model = Model(inputs=model_VGGFace.input,
outputs=[out1, out2])
return custom_vgg_model
'''
实现自己的网络机构
'''
#接下来就是考虑什么模型可以来做这个人脸预测了,测试vgg_face,senet50
from keras_vggface.vggface import VGGFace
from keras.layers import *
from lin import categorical_focal_loss
vgg_model = VGGFace(include_top=False,
model='senet50',
input_shape=(224, 224, 3),
weights=None)
# 必须使用该方法下载模型，然后加载
from flyai.utils import remote_helper
path = remote_helper.get_remote_date(
'https://www.flyai.com/m/rcmalli_vggface_tf_notop_senet50.h5')
vgg_model.load_weights(path)
x = vgg_model.output
x = Flatten()(x)
x = BatchNormalization()(x)
x = Dense(128, activation='relu')(x)
x = Dropout(rate=0.5)(x)
x = Dense(128, activation='relu')(x)
x = Dropout(rate=0.5)(x)
o = Dense(10, activation='softmax', name='prediction')(x)
seque = Keras_Model(inputs=vgg_model.input, outputs=o)
#学习率设置
from keras.optimizers import *
from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint
sgd = SGD(lr=0.0001)
seque.compile(loss=categorical_focal_loss(gamma=2., alpha=0.5),
def run(img_dirA, img_dirB, TOTAL_ITERS = 40000):
global model, vggface
img_dirA_bm_eyes = f"{img_dirA}/binary_masks_eyes2"
img_dirB_bm_eyes = f"{img_dirB}/binary_masks_eyes2"
# Path to saved model weights
models_dir = "./models"
# Number of CPU cores
num_cpus = os.cpu_count()
# Input/Output resolution
RESOLUTION = 64 # 64x64, 128x128, 256x256
assert (RESOLUTION % 64) == 0, "RESOLUTION should be 64, 128, or 256."
# Batch size
batchSize = 8
assert (batchSize != 1 and batchSize % 2 == 0) , "batchSize should be an even number."
# Use motion blurs (data augmentation)
# set True if training data contains images extracted from videos
use_da_motion_blur = False
# Use eye-aware training
# require images generated from prep_binary_masks.ipynb
use_bm_eyes = True
# Probability of random color matching (data augmentation)
prob_random_color_match = 0.5
da_config = {
"prob_random_color_match": prob_random_color_match,
"use_da_motion_blur": use_da_motion_blur,
"use_bm_eyes": use_bm_eyes
}
# Architecture configuration
arch_config = {}
arch_config['IMAGE_SHAPE'] = (RESOLUTION, RESOLUTION, 3)
arch_config['use_self_attn'] = True
arch_config['norm'] = "instancenorm" # instancenorm, batchnorm, layernorm, groupnorm, none
arch_config['model_capacity'] = "standard" # standard, lite
# VGGFace ResNet50
vggface = VGGFace(include_top=False, model='resnet50', input_shape=(224, 224, 3))
# Loss function weights configuration
loss_weights = {}
loss_weights['w_D'] = 0.1 # Discriminator
loss_weights['w_recon'] = 1. # L1 reconstruction loss
loss_weights['w_edge'] = 0.1 # edge loss
loss_weights['w_eyes'] = 30. # reconstruction and edge loss on eyes area
loss_weights['w_pl'] = (0.01, 0.1, 0.3, 0.1) # perceptual loss (0.003, 0.03, 0.3, 0.3)
# Init. loss config.
loss_config = {}
loss_config["gan_training"] = "mixup_LSGAN" # "mixup_LSGAN" or "relativistic_avg_LSGAN"
loss_config['use_PL'] = False
loss_config["PL_before_activ"] = False
loss_config['use_mask_hinge_loss'] = False
loss_config['m_mask'] = 0.
loss_config['lr_factor'] = 1.
loss_config['use_cyclic_loss'] = False
from networks.faceswap_gan_model import FaceswapGANModel
model = FaceswapGANModel(**arch_config)
model.load_weights(path=models_dir)
from colab_demo.vggface_models import RESNET50
vggface = RESNET50(include_top=False, weights=None, input_shape=(224, 224, 3))
vggface.load_weights("rcmalli_vggface_tf_notop_resnet50.h5")
#vggface.summary()
model.build_pl_model(vggface_model=vggface, before_activ=loss_config["PL_before_activ"])
model.build_train_functions(loss_weights=loss_weights, **loss_config)
from data_loader.data_loader import DataLoader
# Create ./models directory
Path(f"models").mkdir(parents=True, exist_ok=True)
# Get filenames
faces_A = f"{img_dirA}/raw_faces"
faces_B = f"{img_dirB}/raw_faces"
train_A = glob.glob(f"{faces_A}/*.*")
train_B = glob.glob(f"{faces_B}/*.*")
train_AnB = train_A + train_B
assert len(train_A), f"No image found in {faces_A}"
assert len(train_B), f"No image found in {faces_B}"
print ("Number of images in folder A: " + str(len(train_A)))
print ("Number of images in folder B: " + str(len(train_B)))
if use_bm_eyes:
assert len(glob.glob(img_dirA_bm_eyes+"/*.*")), "No binary mask found in " + str(img_dirA_bm_eyes)
assert len(glob.glob(img_dirB_bm_eyes+"/*.*")), "No binary mask found in " + str(img_dirB_bm_eyes)
assert len(glob.glob(img_dirA_bm_eyes+"/*.*")) == len(train_A), \
"Number of faceA images does not match number of their binary masks. Can be caused by any none image file in the folder."
assert len(glob.glob(img_dirB_bm_eyes+"/*.*")) == len(train_B), \
"Number of faceB images does not match number of their binary masks. Can be caused by any none image file in the folder."
def show_loss_config(loss_config):
for config, value in loss_config.items():
print(f"{config} = {value}")
def reset_session(save_path):
global model, vggface
global train_batchA, train_batchB
model.save_weights(path=save_path)
del model
del vggface
del train_batchA
del train_batchB
K.clear_session()
model = FaceswapGANModel(**arch_config)
model.load_weights(path=save_path)
vggface = VGGFace(include_top=False, model='resnet50', input_shape=(224, 224, 3))
model.build_pl_model(vggface_model=vggface, before_activ=loss_config["PL_before_activ"])
train_batchA = DataLoader(train_A, train_AnB, batchSize, img_dirA_bm_eyes,
RESOLUTION, num_cpus, K.get_session(), **da_config)
train_batchB = DataLoader(train_B, train_AnB, batchSize, img_dirB_bm_eyes,
RESOLUTION, num_cpus, K.get_session(), **da_config)
# Start training
t0 = time.time()
# This try/except is meant to resume training that was accidentally interrupted
try:
gen_iterations
print(f"Resume training from iter {gen_iterations}.")
except:
gen_iterations = 0
errGA_sum = errGB_sum = errDA_sum = errDB_sum = 0
errGAs = {}
errGBs = {}
# Dictionaries are ordered in Python 3.6
for k in ['ttl', 'adv', 'recon', 'edge', 'pl']:
errGAs[k] = 0
errGBs[k] = 0
display_iters = 300
backup_iters = 5000
global train_batchA, train_batchB
train_batchA = DataLoader(train_A, train_AnB, batchSize, img_dirA_bm_eyes,
RESOLUTION, num_cpus, K.get_session(), **da_config)
train_batchB = DataLoader(train_B, train_AnB, batchSize, img_dirB_bm_eyes,
RESOLUTION, num_cpus, K.get_session(), **da_config)
clear_output()
loss_config['use_PL'] = True
loss_config['use_mask_hinge_loss'] = False
loss_config['m_mask'] = 0.0
reset_session(models_dir)
print("Building new loss funcitons...")
show_loss_config(loss_config)
model.build_train_functions(loss_weights=loss_weights, **loss_config)
print("Done.")
while gen_iterations <= TOTAL_ITERS:
# Loss function automation
if gen_iterations == (TOTAL_ITERS//5 - display_iters//2):
clear_output()
loss_config['use_PL'] = True
loss_config['use_mask_hinge_loss'] = False
loss_config['m_mask'] = 0.0
reset_session(models_dir)
print("Building new loss funcitons...")
show_loss_config(loss_config)
model.build_train_functions(loss_weights=loss_weights, **loss_config)
print("Done.")
elif gen_iterations == (TOTAL_ITERS//5 + TOTAL_ITERS//10 - display_iters//2):
clear_output()
loss_config['use_PL'] = True
loss_config['use_mask_hinge_loss'] = True
loss_config['m_mask'] = 0.5
reset_session(models_dir)
print("Building new loss funcitons...")
show_loss_config(loss_config)
model.build_train_functions(loss_weights=loss_weights, **loss_config)
print("Complete.")
elif gen_iterations == (2*TOTAL_ITERS//5 - display_iters//2):
clear_output()
loss_config['use_PL'] = True
loss_config['use_mask_hinge_loss'] = True
loss_config['m_mask'] = 0.2
reset_session(models_dir)
print("Building new loss funcitons...")
show_loss_config(loss_config)
model.build_train_functions(loss_weights=loss_weights, **loss_config)
print("Done.")
elif gen_iterations == (TOTAL_ITERS//2 - display_iters//2):
clear_output()
loss_config['use_PL'] = True
loss_config['use_mask_hinge_loss'] = True
loss_config['m_mask'] = 0.4
reset_session(models_dir)
print("Building new loss funcitons...")
show_loss_config(loss_config)
model.build_train_functions(loss_weights=loss_weights, **loss_config)
print("Done.")
elif gen_iterations == (2*TOTAL_ITERS//3 - display_iters//2):
clear_output()
loss_config['use_PL'] = True
loss_config['use_mask_hinge_loss'] = False
loss_config['m_mask'] = 0.
loss_config['lr_factor'] = 0.3
reset_session(models_dir)
print("Building new loss funcitons...")
show_loss_config(loss_config)
model.build_train_functions(loss_weights=loss_weights, **loss_config)
print("Done.")
elif gen_iterations == (8*TOTAL_ITERS//10 - display_iters//2):
clear_output()
model.decoder_A.load_weights("models/decoder_B.h5") # swap decoders
model.decoder_B.load_weights("models/decoder_A.h5") # swap decoders
loss_config['use_PL'] = True
loss_config['use_mask_hinge_loss'] = True
loss_config['m_mask'] = 0.1
loss_config['lr_factor'] = 0.3
reset_session(models_dir)
print("Building new loss funcitons...")
show_loss_config(loss_config)
model.build_train_functions(loss_weights=loss_weights, **loss_config)
print("Done.")
elif gen_iterations == (9*TOTAL_ITERS//10 - display_iters//2):
clear_output()
loss_config['use_PL'] = True
loss_config['use_mask_hinge_loss'] = False
loss_config['m_mask'] = 0.0
loss_config['lr_factor'] = 0.1
reset_session(models_dir)
print("Building new loss funcitons...")
show_loss_config(loss_config)
model.build_train_functions(loss_weights=loss_weights, **loss_config)
print("Done.")
if gen_iterations == 5:
print ("working.")
# Train dicriminators for one batch
data_A = train_batchA.get_next_batch()
data_B = train_batchB.get_next_batch()
errDA, errDB = model.train_one_batch_D(data_A=data_A, data_B=data_B)
errDA_sum +=errDA[0]
errDB_sum +=errDB[0]
# Train generators for one batch
data_A = train_batchA.get_next_batch()
data_B = train_batchB.get_next_batch()
errGA, errGB = model.train_one_batch_G(data_A=data_A, data_B=data_B)
errGA_sum += errGA[0]
errGB_sum += errGB[0]
for i, k in enumerate(['ttl', 'adv', 'recon', 'edge', 'pl']):
errGAs[k] += errGA[i]
errGBs[k] += errGB[i]
gen_iterations+=1
# Visualization
if gen_iterations % display_iters == 0:
clear_output()
# Display loss information
show_loss_config(loss_config)
print("----------")
print('[iter %d] Loss_DA: %f Loss_DB: %f Loss_GA: %f Loss_GB: %f time: %f'
% (gen_iterations, errDA_sum/display_iters, errDB_sum/display_iters,
errGA_sum/display_iters, errGB_sum/display_iters, time.time()-t0))
print("----------")
print("Generator loss details:")
print(f'[Adversarial loss]')
print(f'GA: {errGAs["adv"]/display_iters:.4f} GB: {errGBs["adv"]/display_iters:.4f}')
print(f'[Reconstruction loss]')
print(f'GA: {errGAs["recon"]/display_iters:.4f} GB: {errGBs["recon"]/display_iters:.4f}')
print(f'[Edge loss]')
print(f'GA: {errGAs["edge"]/display_iters:.4f} GB: {errGBs["edge"]/display_iters:.4f}')
if loss_config['use_PL'] == True:
print(f'[Perceptual loss]')
try:
print(f'GA: {errGAs["pl"][0]/display_iters:.4f} GB: {errGBs["pl"][0]/display_iters:.4f}')
except:
print(f'GA: {errGAs["pl"]/display_iters:.4f} GB: {errGBs["pl"]/display_iters:.4f}')
# Display images
print("----------")
wA, tA, _ = train_batchA.get_next_batch()
wB, tB, _ = train_batchB.get_next_batch()
print("Transformed (masked) results:")
showG(tA, tB, model.path_A, model.path_B, batchSize)
print("Masks:")
showG_mask(tA, tB, model.path_mask_A, model.path_mask_B, batchSize)
print("Reconstruction results:")
showG(wA, wB, model.path_bgr_A, model.path_bgr_B, batchSize)
errGA_sum = errGB_sum = errDA_sum = errDB_sum = 0
for k in ['ttl', 'adv', 'recon', 'edge', 'pl']:
errGAs[k] = 0
errGBs[k] = 0
# Save models
model.save_weights(path=models_dir)
# Backup models
if gen_iterations % backup_iters == 0:
bkup_dir = f"{models_dir}/backup_iter{gen_iterations}"
Path(bkup_dir).mkdir(parents=True, exist_ok=True)
model.save_weights(path=bkup_dir)
