def setup(self):
distorted_A, fake_A, fake_sz64_A, mask_A, self.path_A, self.path_mask_A, self.path_abgr_A, self.path_bgr_A = self.cycle_variables(self.model.netGA)
distorted_B, fake_B, fake_sz64_B, mask_B, self.path_B, self.path_mask_B, self.path_abgr_B, self.path_bgr_B = self.cycle_variables(self.model.netGB)
real_A = Input(shape=self.model.img_shape)
real_B = Input(shape=self.model.img_shape)
if self.use_lsgan:
self.loss_fn = lambda output, target : K.mean(K.abs(K.square(output-target)))
else:
self.loss_fn = lambda output, target : -K.mean(K.log(output+1e-12)*target+K.log(1-output+1e-12)*(1-target))
# ========== Define Perceptual Loss Model==========
if self.use_perceptual_loss:
from keras.models import Model
from keras_vggface.vggface import VGGFace
vggface = VGGFace(include_top=False, model='resnet50', input_shape=(224, 224, 3))
vggface.trainable = False
out_size55 = vggface.layers[36].output
out_size28 = vggface.layers[78].output
out_size7 = vggface.layers[-2].output
vggface_feat = Model(vggface.input, [out_size55, out_size28, out_size7])
vggface_feat.trainable = False
else:
vggface_feat = None
loss_DA, loss_GA = self.define_loss(self.model.netDA, real_A, fake_A, fake_sz64_A, distorted_A, vggface_feat)
loss_DB, loss_GB = self.define_loss(self.model.netDB, real_B, fake_B, fake_sz64_B, distorted_B, vggface_feat)
if self.use_mask_refinement:
loss_GA += 1e-3 * K.mean(K.square(mask_A))
loss_GB += 1e-3 * K.mean(K.square(mask_B))
else:
loss_GA += 3e-3 * K.mean(K.abs(mask_A))
loss_GB += 3e-3 * K.mean(K.abs(mask_B))
w_fo = 0.01
loss_GA += w_fo * K.mean(self.first_order(mask_A, axis=1))
loss_GA += w_fo * K.mean(self.first_order(mask_A, axis=2))
loss_GB += w_fo * K.mean(self.first_order(mask_B, axis=1))
loss_GB += w_fo * K.mean(self.first_order(mask_B, axis=2))
weightsDA = self.model.netDA.trainable_weights
weightsGA = self.model.netGA.trainable_weights
weightsDB = self.model.netDB.trainable_weights
weightsGB = self.model.netGB.trainable_weights
# Adam(..).get_updates(...)
training_updates = Adam(lr=self.lrD, beta_1=0.5).get_updates(weightsDA,[],loss_DA)
self.netDA_train = K.function([distorted_A, real_A],[loss_DA], training_updates)
training_updates = Adam(lr=self.lrG, beta_1=0.5).get_updates(weightsGA,[], loss_GA)
self.netGA_train = K.function([distorted_A, real_A], [loss_GA], training_updates)
training_updates = Adam(lr=self.lrD, beta_1=0.5).get_updates(weightsDB,[],loss_DB)
self.netDB_train = K.function([distorted_B, real_B],[loss_DB], training_updates)
training_updates = Adam(lr=self.lrG, beta_1=0.5).get_updates(weightsGB,[], loss_GB)
self.netGB_train = K.function([distorted_B, real_B], [loss_GB], training_updates)
def genderModel(input_shape=(224, 224, 3), model_type='vgg16'):
input_shape = input_shape
model_type = model_type
vggface_model = VGGFace(include_top=False, model=model_type, weights='vggface',\
input_shape=input_shape)
print("Base VGG model summary.")
vggface_model.summary()
return vggface_model
def get_embeddings(filenames):
faces = [extract_face(f) for f in filenames]
samples = asarray(faces, 'float32')
samples = preprocess_input(samples, version=2)
model = VGGFace(model='resnet50',
include_top=False,
input_shape=(200, 200, 3),
pooling='avg')
yhat = model.predict(samples)
return yhat
def define_model(self,
hidden_dim=128,
drop_rate=0.0,
freeze_backbone=True):
if self.model_type == 'vgg16_fc6':
vgg_model = VGGFace(model='vgg16',
include_top=True,
input_shape=(224, 224, 3))
last_layer = vgg_model.get_layer('fc6').output
flatten = Activation('relu')(last_layer)
else:
vgg_model = VGGFace(model=self.model_type,
include_top=False,
input_shape=(224, 224, 3))
last_layer = vgg_model.output
flatten = Flatten()(last_layer)
if freeze_backbone:
for layer in vgg_model.layers:
layer.trainable = False
def block(flatten, name):
x = Dense(hidden_dim, name=name + '_fc1')(flatten)
x = BatchNormalization(name=name + '_bn1')(x)
x = Activation('relu', name=name + '_act1')(x)
x = Dropout(drop_rate)(x)
x = Dense(hidden_dim, name=name + '_fc2')(x)
x = BatchNormalization(name=name + '_bn2')(x)
x = Activation('relu', name=name + '_act2')(x)
x = Dropout(drop_rate)(x)
return x
x = block(flatten, name='bmi')
out_bmi = Dense(1, activation='linear', name='bmi')(x)
x = block(flatten, name='age')
out_age = Dense(1, activation='linear', name='age')(x)
x = block(flatten, name='sex')
out_sex = Dense(1, activation='sigmoid', name='sex')(x)
custom_vgg_model = Model(vgg_model.input, [out_bmi, out_age, out_sex])
custom_vgg_model.compile('adam', {
'bmi': 'mae',
'age': 'mae',
'sex': 'binary_crossentropy'
}, {'sex': 'accuracy'},
loss_weights={
'bmi': 0.8,
'age': 0.1,
'sex': 0.1
})
self.model = custom_vgg_model
def __init__(self, layer='fc6'):
logging.info('loading VGG face')
self.layer = layer
vgg_face = VGGFace()
self.model = Model(inputs=vgg_face.layers[0].input,
outputs=vgg_face.get_layer(self.layer).output)
session = K.get_session()
K.set_session(session)
def get_model_scores(faces):
samples = asarray(faces, 'float32')
# prepare the data for the model
samples = preprocess_input(samples, version=2)
# create a vggface model object
model = VGGFace(model='vgg16',
include_top=False,
input_shape=(224, 224, 3),
pooling='avg')
# perform prediction
return model.predict(samples)
def getVGGFace(configs):
vgg_model = VGGFace(include_top=False, input_shape=configs["imgSize"])
last_layer = vgg_model.get_layer('conv5_3').output
model = Model(inputs=vgg_model.input, outputs=last_layer)
print("---------Encoder VGGFace---------")
model.summary()
print("---------Encoder VGGFace ---------")
return model
def bulid_model():
vgg_model = VGGFace(include_top=False, input_shape=(64, 64, 3))
last_layer = vgg_model.get_layer('pool5').output
x = Flatten(name='flatten')(last_layer)
x = Dense(HIDDEN_DIM, activation='relu', name='fc6')(x)
#x = Dense(HIDDEN_DIM, activation='relu', name='fc7')(x)
x = Dropout(0.4)(x)
out = Dense(NB_CLASS, activation='softmax', name='fc8')(x)
custom_vgg_model = Model(vgg_model.input, out)
custom_vgg_model.summary()
return custom_vgg_model
def create_face_network(nb_class=2, hidden_dim=512, shape=(224, 224, 3)):
model = VGGFace(include_top=False, input_shape=shape)
last_layer = model.get_layer('pool5').output
x = Flatten(name='flatten')(last_layer)
x = Dense(hidden_dim, activation='relu', name='fc6')(x)
x = Dense(hidden_dim, activation='relu', name='fc7')(x)
out = Dense(nb_class, activation='softmax', name='fc8')(x)
custom_vgg_model = Model(model.input, out)
print(custom_vgg_model.summary())
return custom_vgg_model
def extract_vgg_features(images_folder, output_folder=None):
if output_folder == None:
output_folder = os.path.join(images_folder + "_", "vgg_features")
# Convolution Features
vgg_features = VGGFace(include_top=False,
input_shape=(224, 224, 3),
pooling='avg') # pooling: None, avg or max
categories = sorted([
f for f in os.listdir(images_folder) if
os.path.isdir(os.path.join(images_folder, f)) and not f.startswith('.')
],
key=lambda f: f.lower())
for category in categories:
if not os.path.exists(os.path.join(output_folder, category)):
os.makedirs(os.path.join(output_folder, category))
videos = sorted([
f for f in os.listdir(os.path.join(images_folder, category))
if os.path.isdir(os.path.join(images_folder, category, f))
and not f.startswith('.')
],
key=lambda f: f.lower())
for video in videos:
images = sorted([
f for f in os.listdir(
os.path.join(images_folder, category, video)) if
os.path.isfile(os.path.join(images_folder, category, video, f))
and not f.startswith('.')
],
key=lambda f: f.lower())
features = []
for i in images:
img_path = os.path.join(images_folder, category, video, i)
img = image.load_img(img_path, target_size=(224, 224))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = utils.preprocess_input(
x, version=1) # version=1 (VGG16) or version=2 (RESNET50)
features.append(vgg_features.predict(x))
print("Predicted %s" % img_path)
features = np.concatenate(tuple(features), axis=0)
with open(os.path.join(output_folder, category, video + ".csv"),
"w+") as write_file:
writer = csv.writer(write_file)
writer.writerows(features)
a = 0
def valid_entrant(img):
model = VGGFace(input_shape=(224, 224,
3)) # (weights='vggface') for default
model.load_weights("model_weights.h5") # custom trained
im = image.img_to_array(img)
im2 = transform.resize(im, (224, 224, 3)) #resize input image
preds = model.predict(im2)
x = (preds[0][0]) # class label of our most likely prediction
if x in permitted:
return True
else:
return False
def face_process():
#vgg_features = VGGFace(include_top=False, input_shape=(224, 224, 3), pooling='avg') ## doesn't seem to be required.
print("* Loading model")
model = VGGFace(model='vgg16')
print("* Model loaded")
while True:
queue = db.lrange(settings.FACE_QUEUE, 0, settings.BATCH_SIZE - 1)
imageIDs = []
batch = None
for q in queue:
# deserialize the object and obtain the input image
q = json.loads(q.decode("utf-8"))
image = helpers.base64_decode_image(
q["image"], settings.IMAGE_DTYPE,
(1, settings.IMAGE_HEIGHT, settings.IMAGE_WIDTH,
settings.IMAGE_CHANS))
# check to see if the batch list is None
if batch is None:
batch = image
# otherwise, stack the data
else:
batch = np.vstack([batch, image])
# update the list of image IDs
imageIDs.append(q["id"])
# check to see if we need to process the batch
if len(imageIDs) > 0:
# classify the batch
print("* Batch size: {}".format(batch.shape))
preds = model.predict(batch)
results = utils.decode_predictions(preds)
#print(results) ## this comes back with something so its the below structure giving me an issue.
# [[["b'A.J._Buckley'", 0.9768057], ["b'David_Denman'", 0.0013909286], ["b'Carmine_Giovinazzo'", 0.0010687601], ["b'Robert_Buckley'", 0.00093060045], ["b'Eddie_Cahill'", 0.00044030472]]]
for (imageID, resultSet) in zip(imageIDs, results):
print("imageID", imageID, resultSet)
## imageID 63350aef-0ec3-4e1d-be99-3db36013a6d7
output = []
for (label, prob) in resultSet:
r = {"label": label, "probability": float(prob)}
output.append(r)
db.set(imageID, json.dumps(output))
# remove the set of images from our queue
db.ltrim(settings.FACE_QUEUE, len(imageIDs), -1)
# sleep for a small amount
time.sleep(settings.SERVER_SLEEP)
def __init__(self, shape, out_dim, hid_dim):
self.hid_dim = hid_dim
super(FaceVGG16, self).__init__(shape, out_dim, "FaceVGG16")
vgg_model = VGGFace(model='vgg16',
include_top=False,
input_shape=self.shape)
last_layer = vgg_model.get_layer('pool5').output
x = Flatten(name='flatten')(last_layer)
x = Dense(self.hid_dim, activation='relu', name='fc6')(x)
x = Dense(self.hid_dim, activation='relu', name='fc7')(x)
out = Dense(self.out_dim, activation='softmax', name='fc8')(x)
self.model = Model(vgg_model.input, out)(x)
def __init__(self, shape, out_dim):
super(RESNET50, self).__init__(shape, out_dim, "RESNET50")
vgg_model = VGGFace(model='resnet50',
include_top=False,
input_shape=self.shape)
last_layer = vgg_model.get_layer('avg_pool').output
x = Flatten(name='flatten')(last_layer)
out = Dense(self.out_dim, activation='softmax', name='classifier')(x)
self.model = Model(vgg_model.input, out)
self.model.compile(loss='categorical_crossentropy',
optimizer=Adadelta(lr=1.0),
metrics=['accuracy'])
def __init__(self):
self.model = VGGFace(model='resnet50')
feature_layer_name = 'flatten_1'
feature_layer = self.model.get_layer(
feature_layer_name) # 2048-dimensional feature vector
self.feature_descriptor_model = Model(self.model.input,
feature_layer.output)
self.model.summary()
self.labels = load_labels()
self.images_references = load_image_references()
def senet50(self):
senet50_layer = VGGFace(include_top=False,
model='senet50',
weights='vggface',
input_shape=(self.height, self.width,
self.channels))
senet50_layer.trainable = False
# senet50_layer.summary()
return senet50_layer
def getBaseModel():
#base_model = InceptionV3(weights='imagenet', include_top=False)
#base_model = VGGFace(include_top=False, model='vgg16', input_shape=(224, 224, 3), pooling='max')
#base_model = VGGFace(include_top=False, model='vgg16', pooling='avg')
# Layer Features
layer_name = 'pool4'
vgg_model = VGGFace(input_shape=(224, 224, 3))
out = vgg_model.get_layer(layer_name).output
base_model = Model(vgg_model.input, out)
return base_model
def model_graph(self):
if self.model == 'vgg16':
model = VGGFace(model='vgg16')
elif self.model == 'resnet50':
model = VGGFace(model='resnet50')
elif self.model == 'senet50':
model = VGGFace(model='senet50')
GraphsMkdir().check()
plot_model(model,
to_file='../graphs/' + self.model + '.png',
show_shapes=True)
def get_embeddings(filenames): # extract faces faces = [extract_face(f) for f in filenames] # convert into an array of samples samples = asarray(faces, 'float32') # prepare the face for the model, e.g. center pixels samples = preprocess_input(samples, version=2) # create a vggface model model = VGGFace(model='resnet50', include_top=False, input_shape=(224, 224, 3), pooling='avg') # perform prediction yhat = model.predict(samples) return yhat
def VGGFACE(self, input_tensor=None, input_shape=(224, 224, 3)):
vggface = VGGFace(input_tensor=input_tensor,
model='vgg16',
include_top=False,
input_shape=input_shape)
outputs = []
for l in self.vggface_feat_layers:
outputs.append(vggface.get_layer(l).output)
model = tf.keras.Model(inputs=vggface.input, outputs=outputs)
return model
def load_model():
global MODEL
if MODEL is None:
MODEL = VGGFace(model='senet50',
include_top=False,
input_shape=(224, 224, 3),
pooling=None)
output = MODEL.get_layer('add_16').output
x1 = GlobalAveragePooling2D()(output)
x2 = GlobalMaxPooling2D()(output)
x = Concatenate()([x1, x2])
MODEL = Model(MODEL.input, x)
return MODEL
def __init__(self):
# database path
self.db_path = os.path.join(path, 'database', 'db_enc.pkl')
# create a vggface model
self.model = VGGFace(model='resnet50',
include_top=False,
input_shape=(224, 224, 3),
pooling='avg')
# loading database
if (os.path.exists(self.db_path)):
self.database = pk.load(open(self.db_path, "rb"))
else:
self.database = []
def vgg16F_fe(img_input):
# net = preprocess_input(img_input)
from keras_vggface.vggface import VGGFace
vgg_model = VGGFace(include_top=False,
input_tensor=img_input,
pooling='avg')
#vgg_model.layers.pop()
last_layer = vgg_model.get_layer('pool5').output
x = Flatten(name='flatten')(last_layer)
x = Dense(1024, activation='relu', trainable=True)(x)
x = Dense(512, activation='relu', trainable=True)(x)
model = dnn.Model(input=vgg_model.input, output=x)
return model.layers[-1].output
def get_deep_feature(x):
model = VGGFace(include_top=False,
input_shape=(224, 224, 3),
pooling='avg') # pooling: None, avg or max
output = model.get_layer('conv5_3').output
output = GlobalAveragePooling2D()(output)
feature_model = Model(inputs=model.input, outputs=output)
x = utils.preprocess_input(x, version=1) # or version=2
x = feature_model.predict(x)
return x
def __init__(self, layer='fc6', model='vgg16'):
self.model = model
self.layer = layer
# Get model with pretrained weights. model: vgg16, resnet50
vgg_model = VGGFace(model=model)
if self.model == 'vgg16':
# We'll extract features at the fc6 layer
self.model = Model(inputs=vgg_model.input,
outputs=vgg_model.get_layer(layer).output)
elif self.model == 'resnet50':
resent_out = vgg_model.get_layer(layer).output
out = Flatten(name='flatten')(resent_out)
self.model = Model(inputs=vgg_model.input, outputs=out)
def load_recognizer_keras(name):
K.set_learning_phase(0)
if name == 'vgg16':
model_name, layer = 'vgg16', 'fc7/relu'
elif name == 'resnet50':
model_name, layer = 'resnet50', 'avg_pool'
model = VGGFace(model=model_name)
out = model.get_layer(layer).output
cnn_model = Model(model.input, out)
cnn_model.summary()
return cnn_model
def __init__(self, model_name, pooling, image_size, mtcnn_model_path,
vgg_model_path):
self.model_name = model_name
self.pooling = pooling
self.image_size = image_size
self.mtcnn_model_path = mtcnn_model_path
self.vgg_model_path = vgg_model_path
self.model = VGGFace(model=self.model_name,
include_top=False,
input_shape=(self.image_size, self.image_size, 3),
pooling=self.pooling,
weights_path=self.vgg_model_path)
self.detector = MTCNN(weights_file=self.mtcnn_model_path)
def build_resnet50(self):
resnet50 = VGGFace(include_top = False, model = 'resnet50', weights = 'vggface', input_shape = (self.height, self.width, self.channels))
# Make trainable as False
resnet50.trainable = False
for j in resnet50.layers:
j.trainable = False
# resnet50.summary()
return resnet50
def create_model(model_name):
# Layer Features
if model_name == 'vgg16':
layer_name = 'fc7/relu'
elif model_name == 'resnet50':
layer_name = 'flatten_1'
else:
raise Exception('Model name not recognized!')
model = VGGFace(model=model_name)
out = model.get_layer(layer_name).output
return Model(model.input, out)
def build_intermediate_vggface_model(self):
vggface = VGGFace(input_shape=self.input_shape,
weights='vggface',
include_top=False)
vggface.trainable = False
layer_names = ['conv1_2', 'conv2_2', 'conv3_3', 'conv4_3', 'conv5_3']
intermediate_outputs = [
vggface.get_layer(layer_name).output for layer_name in layer_names
]
self.intermediate_vggface = Model(vggface.input,
intermediate_outputs,
name='intermediate_vggface')
return self.intermediate_vggface
def build_vgg16(self):
vgg16 = VGGFace(include_top = False, model = 'vgg16', weights = 'vggface', input_shape = (self.height, self.width, self.channels))
# Make trainable as False
vgg16.trainable = False
for i in vgg16.layers:
i.trainable = False
# vgg16.summary()
return vgg16