def percept_loss(input_image, reconstructed_image):
vggface = VGGFace(include_top=False, input_shape=x_dim, model='vgg16')
vgg_layers = ['conv1_1']
outputs = [vggface.get_layer(l).output for l in vgg_layers]
model = Model(inputs=vggface.input, outputs=outputs)
for layer in model.layers:
layer.trainable = False
input_image *= 255.0
reconstructed_image *= 255.0
input_image = preprocess_input(input_image, mode='tf', data_format='channels_last')
reconstructed_image = preprocess_input(reconstructed_image, mode='tf', data_format='channels_last')
h1_list = model(input_image)
h2_list = model(reconstructed_image)
if not isinstance(h1_list, list):
h1_list = [h1_list]
h2_list = [h2_list]
p_loss = 0.0
for h1, h2 in zip(h1_list, h2_list):
h1 = K.batch_flatten(h1)
h2 = K.batch_flatten(h2)
p_loss += K.mean(K.square(h1 - h2), axis=-1)
return gamma * p_loss
def model_from_vgg(last_layer='pool4'):
"""
returns a neural network with layers upto <last_layer> from vgg16
with the weights for the vggface layers preloaded
"""
vgg_model = VGGFace(model='vgg16', include_top=False, input_shape=(224, 224, 3))
X = vgg_model.get_layer(last_layer).output
layer_shape = vgg_model.get_layer(last_layer).output_shape
n_encoder_layers = int(np.log2(224/layer_shape[2]))
for n in range(n_encoder_layers):
X = Conv2DTranspose(int(layer_shape[3]/(2**(n+1))), (3, 3), activation='relu', padding='same', name='deconv'+str(n+1))(X)
X = UpSampling2D(size=(2, 2), interpolation='bilinear', name='unpool'+str(n+1))(X)
mask = Conv2D(1, (3, 3), activation='sigmoid', padding='same', name='mask')(X)
custom_model = Model(vgg_model.input, mask)
return custom_model
def get_classification_face_match(target_image):
nb_class = 2
hidden_dim = 512
vggmodel = VGGFace(include_top=False, input_shape=(224, 224, 3))
last_layer = vggmodel.get_layer('pool5').output
x = Flatten(name='flatten')(last_layer)
x = Dense(hidden_dim, activation='relu', name='fc6')(x)
x = Dense(hidden_dim, activation='relu', name='fc7')(x)
out = Dense(nb_class, activation='softmax', name='fc8')(x)
identity_model = Model(vggmodel.input, out)
## Train the model
##Predict
img = image.load_img(target_image, target_size=(224, 224))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = utils.preprocess_input(x, version=1)
preds = identity_model.predict(x)
print('\n', "VGG16")
print('\n', preds)
print('\n', 'Predicted:', utils.decode_predictions(preds))
return utils.decode_predictions(preds)[0][0][0]
# image = combine_images(generated_images)
# image = image*127.5+127.5
# Image.fromarray(image.astype(np.uint8)).save(
# # "./image_result/generated_image.png")
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument("--mode", type=str)
parser.add_argument("--batch_size", type=int, default=128)
parser.add_argument("--nice", dest="nice", action="store_true")
parser.set_defaults(nice=False)
args = parser.parse_args()
return args
if __name__ == "__main__":
args = get_args()
if args.mode == "train":
image_input = Input(shape=(3, 112, 96))
vgg_model = VGGFace(input_tensor=image_input,
include_top=False,
pooling='avg') # pooling: None, avg or max
out = vgg_model.get_layer('pool5').output
vgg_conv = Model(image_input, out)
train(vgg_conv)
elif args.mode == "generate":
generate(BATCH_SIZE=args.batch_size, nice=args.nice)
elif args.mode == "mse":
MSE(BATCH_SIZE=args.batch_size)
def train(BATCH_SIZE):
# Load the training data
print('Data loading..')
X_train, y_train, X_test, y_test = Data.loadData('data.h5')
print('Data Loaded. Now normalizing..')
X_train = (X_train.astype(np.float32) - 127.5) / 127.5
y_train = (y_train.astype(np.float32) - 127.5) / 127.5
print('Data Normalized.')
# Optimization setting RMSprop(lr=0.0001, rho=0.9, epsilon=1e-08, decay=0.0)
d_optim = SGD(lr=0.0001, decay=1e-6, momentum=0.9, nesterov=True)
g_optim = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)
g_vgg_optim = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True)
# Vgg model goes here
image_input = Input(shape=(112, 96, 3))
vgg_model = VGGFace(input_tensor=image_input,
include_top=False,
pooling='avg') # pooling: None, avg or max
out = vgg_model.get_layer('pool5').output
vgg_conv = Model(image_input, out)
# Generator model goes here
generator = res_net((112, 96, 3))
# Generator = cnn_model()
generator.compile(loss='mean_squared_error', optimizer=g_optim)
# Discriminative model goes here
discriminator = discriminator_model()
discriminator.trainable = True
discriminator.compile(loss='binary_crossentropy', optimizer=d_optim)
# Gener_VGG model
generator_vgg = \
generator_containing_discriminator(generator, vgg_conv)
generator_vgg.compile(loss=conv_loss, optimizer=g_vgg_optim)
# Gener_Discrim model
generator_discriminator = \
generator_containing_discriminator(generator, discriminator)
generator_discriminator.compile(loss='binary_crossentropy',
optimizer=g_optim)
for epoch in range(100):
print("Epoch is", epoch)
print("Number of batches", int(X_train.shape[0] / BATCH_SIZE))
for index in range(int(X_train.shape[0] / BATCH_SIZE)):
lr_image_batch = X_train[index * BATCH_SIZE:(index + 1) *
BATCH_SIZE]
hr_image_batch = y_train[index * BATCH_SIZE:(index + 1) *
BATCH_SIZE]
generated_images = generator.predict(lr_image_batch, verbose=0)
shape = generated_images.shape
if index % 10 == 0:
image = save_image(generated_images)
image = image * 127.5 + 127.5
#imsave("./image_result/"+ str(epoch)+"_"+ str(index)+ ".png", image)
# imsave("./image_result/"+ str(epoch)+"_"+ str(index)+ ".png", image.astype(np.uint8))
im = Image.fromarray(image.astype(np.uint8))
im.save("./image_result/" + str(epoch) + "_" + str(index) +
".png")
X = np.concatenate((hr_image_batch, generated_images))
y = [1] * BATCH_SIZE + [0] * BATCH_SIZE
if epoch >= 5:
# Discriminative Model Training
d_loss = discriminator.train_on_batch(X, y)
print("batch %d d_loss : %f" % (index, d_loss))
discriminator.trainable = False
# Generator Model Training
g_loss1 = generator.train_on_batch(lr_image_batch,
hr_image_batch)
print("batch %d gene_discri_loss : %f" % (index, g_loss1))
# Generator_Discri Model Training
g_loss2 = generator_discriminator.train_on_batch(
lr_image_batch, [1] * BATCH_SIZE)
discriminator.trainable = True
print("batch %d gene_discri_loss : %f" % (index, g_loss2))
print(' ')
else:
# Discriminative Model Training
d_loss = discriminator.train_on_batch(X, y)
print("batch %d d_loss : %f" % (index, d_loss))
discriminator.trainable = False
# Generator Model Training
g_loss1 = generator.train_on_batch(lr_image_batch,
hr_image_batch)
print("batch %d generator loss : %f" % (index, g_loss1))
# Generator_Discri Model Training
g_loss2 = generator_discriminator.train_on_batch(
lr_image_batch, [1] * BATCH_SIZE)
discriminator.trainable = True
print("batch %d gene_discri_loss : %f" % (index, g_loss2))
# Generate feature labels for the hr_images
labels = vgg_conv.predict(hr_image_batch)
g_loss2 = generator_vgg.train_on_batch(lr_image_batch, labels)
print("batch %d gene_vgg_loss : %f" % (index, g_loss2))
if index % 10 == 9:
generator.save_weights('generator', True)
discriminator.save_weights('discriminator', True)
for i, layer in enumerate(model_F_L.layers):
layer.name = layer.name + '_f_l'
for i, layer in enumerate(model_F_R.layers):
layer.name = layer.name + '_f_r'
# Freeze layers that are not needed to train
for layer in model_M_L.layers[:LayersToFreeze_E]:
layer.trainable = False
for layer in model_M_R.layers[:LayersToFreeze_E]:
layer.trainable = False
for layer in model_F_L.layers[:LayersToFreeze_E]:
layer.trainable = False
for layer in model_F_R.layers[:LayersToFreeze_E]:
layer.trainable = False
last_layer_M_L = model_M_L.get_layer('conv5_3_m_l').output
last_layer_M_R = model_M_R.get_layer('conv5_3_m_r').output
last_layer_F_L = model_F_L.get_layer('conv5_3_f_l').output
last_layer_F_R = model_F_R.get_layer('conv5_3_f_r').output
# Global Average pooling layer at output of 4 networks
modelOutM_L = GlobalAveragePooling2D()(last_layer_M_L)
modelOutM_R = GlobalAveragePooling2D()(last_layer_M_R)
modelOutF_L = GlobalAveragePooling2D()(last_layer_F_L)
modelOutF_R = GlobalAveragePooling2D()(last_layer_F_R)
# Landmark layers
model_land_i = Input((272, ))
model_land_hidden_1 = Dense(272)(model_land_i)
model_land_elev_pred = Dense(NetworkHelper.numElevClasses)(model_land_hidden_1)
model_land_azim_pred = Dense(NetworkHelper.numAzimClasses)(model_land_hidden_1)
from keras.models import Sequential, Model
from keras.layers import Dense, Activation, Dropout, Flatten, Convolution2D, MaxPooling2D, ZeroPadding2D
from keras.optimizers import Adam
from keras_vggface import VGGFace
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ModelCheckpoint, EarlyStopping
device_name = tf.test.gpu_device_name()
vgg_face = VGGFace(include_top=True, input_shape=(224, 224, 3))
for layer in vgg_face.layers:
layer.trainable = False
last_layer = vgg_face.get_layer('fc7/relu').output
out = Dense(1283, activation='softmax', name='fc8')(last_layer)
custom_vgg_face = Model(vgg_face.input, out)
adam = Adam(lr=1e-3, beta_1=0.9, beta_2=0.999)
custom_vgg_face.compile(optimizer=adam,
loss='categorical_crossentropy',
metrics=['accuracy'])
traindir = r"..\data\VGG_DB\trainSet"
valdir = r"..\data\VGG_DB\validSet"
testdir = r"..\data\VGG_DB\TestSet"
batch_size = 64
input_shape = ()
import os
from keras.engine import Model
from keras.layers import Input
from keras_vggface import VGGFace
import numpy as np
from keras.preprocessing import image
from scipy.io import savemat
# FC7 Features
vgg_model = VGGFace() # pooling: None, avg or max
out = vgg_model.get_layer('fc7').output
vgg_model_fc7 = Model(vgg_model.input, out)
def get_features(image_path):
"""
:param image_path(str): Path of a image to extract features with VGG-face.
:return: A 4096 feature vector.
"""
img = image.load_img(image_path, target_size=(224, 224))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
vgg_model_fc7_preds = vgg_model_fc7.predict(x)
#print(vgg_model_fc7_preds[0].shape)
return vgg_model_fc7_preds[0]
def create_all_features(path):
"""
