def extract_PSE_feats(self, x):
'''
:param x: x format is RGB and is value range is [-1,1].
:return: fc1ls: shape, fc1le: expression, pose_model.preds_unNormalized: pose
'''
x = tf.image.resize_images(x, [227, 227])
# x is RGB and is value range is [-1,1].
# first we need to change RGB to BGR;
batch_, height_, width_, nc = x.get_shape().as_list()
R = tf.reshape(x[:, :, :, 0], [batch_, height_, width_, 1])
G = tf.reshape(x[:, :, :, 1], [batch_, height_, width_, 1])
B = tf.reshape(x[:, :, :, 2], [batch_, height_, width_, 1])
x = tf.concat([B, G, R], axis=3)
# second change range [-1,1] to [0,255]
x = (x + 1.0) * 127.5
###################
# Face Pose-Net
###################
try:
net_data = np.load(self.cfg.PAM_frontal_ALexNet_file_path,
encoding="latin1").item()
pose_labels = np.zeros([self.cfg.batch_size, 6])
print('Load ' + self.cfg.PAM_frontal_ALexNet_file_path +
' successful....')
except:
raise Exception('Load ' + self.cfg.PAM_frontal_ALexNet_file_path +
' failed....')
x1 = tf.image.resize_bilinear(x, tf.constant([227, 227],
dtype=tf.int32))
# Image normalization
x1 = x1 / 255. # from [0,255] to [0,1]
# subtract training mean
mean = tf.reshape(self.train_mean_vec, [1, 1, 1, 3])
mean = tf.cast(mean, 'float32')
x1 = x1 - mean
pose_model = Pose_model.Pose_Estimation(x1, pose_labels, 'valid', 0, 1,
1, 0.01, net_data,
self.cfg.batch_size,
self.mean_labels,
self.std_labels)
pose_model._build_graph()
self.pose = pose_model.preds_unNormalized
del net_data
###################
# Shape CNN
###################
x2 = tf.image.resize_bilinear(x, tf.constant([224, 224],
dtype=tf.int32))
x2 = tf.cast(x2, 'float32')
x2 = tf.reshape(x2, [self.cfg.batch_size, 224, 224, 3])
# Image normalization
mean = tf.reshape(self.mean_image_shape, [1, 224, 224, 3])
mean = tf.cast(mean, 'float32')
x2 = x2 - mean
with tf.variable_scope('shapeCNN'):
net_shape = resnet101_shape({'input': x2}, trainable=True)
pool5 = net_shape.layers['pool5']
pool5 = tf.squeeze(pool5)
pool5 = tf.reshape(pool5, [self.cfg.batch_size, -1])
try:
npzfile = np.load(self.cfg.ShapeNet_fc_weights_file_path)
print('Load ' + self.cfg.ShapeNet_fc_weights_file_path +
' successful....')
except:
raise Exception('Load ' +
self.cfg.ShapeNet_fc_weights_file_path +
' failed....')
ini_weights_shape = npzfile['ini_weights_shape']
ini_biases_shape = npzfile['ini_biases_shape']
with tf.variable_scope('shapeCNN_fc1'):
fc1ws = tf.Variable(tf.reshape(ini_weights_shape, [2048, -1]),
trainable=True,
name='weights')
fc1bs = tf.Variable(tf.reshape(ini_biases_shape, [-1]),
trainable=True,
name='biases')
self.fc1ls = tf.nn.bias_add(tf.matmul(pool5, fc1ws), fc1bs)
###################
# Expression CNN
###################
with tf.variable_scope('exprCNN'):
net_expr = resnet101_expr({'input': x2}, trainable=True)
pool5 = net_expr.layers['pool5']
pool5 = tf.squeeze(pool5)
pool5 = tf.reshape(pool5, [self.cfg.batch_size, -1])
try:
npzfile = np.load(self.cfg.ExpNet_fc_weights_file_path)
ini_weights_expr = npzfile['ini_weights_expr']
ini_biases_expr = npzfile['ini_biases_expr']
print('Load ' + self.cfg.ExpNet_fc_weights_file_path +
' successful....')
except:
raise Exception('Load ' +
self.cfg.ExpNet_fc_weights_file_path +
' failed....')
# time.sleep(30)
with tf.variable_scope('exprCNN_fc1'):
fc1we = tf.Variable(tf.reshape(ini_weights_expr, [2048, 29]),
trainable=True,
name='weights')
fc1be = tf.Variable(tf.reshape(ini_biases_expr, [29]),
trainable=True,
name='biases')
self.fc1le = tf.nn.bias_add(tf.matmul(pool5, fc1we), fc1be)
def extract_PSE_feats():
# Prepare data
data_dict = myparse.parse_input(inputlist) # please see input.csv for the input format
print(len(data_dict))
## Pre-processing the images
print('> preproc')
pu.preProcessImage(_tmpdir, data_dict, './', factor, _alexNetSize, output_proc)
# placeholders for the batches
x = tf.placeholder(tf.float32, [FLAGS.batch_size, FLAGS.image_size, FLAGS.image_size, 3])
###################
# Face Pose-Net
###################
net_data = np.load("./fpn_new_model/PAM_frontal_ALexNet_py3.npy").item()
pose_labels = np.zeros([FLAGS.batch_size,6])
x1 = tf.image.resize_bilinear(x, tf.constant([227,227], dtype=tf.int32))
# Image normalization
x1 = x1 / 255. # from [0,255] to [0,1]
# subtract training mean
mean = tf.reshape(train_mean_vec, [1, 1, 1, 3])
mean = tf.cast(mean, 'float32')
x1 = x1 - mean
pose_model = Pose_model.Pose_Estimation(x1, pose_labels, 'valid', 0, 1, 1, 0.01, net_data, FLAGS.batch_size, mean_labels, std_labels)
pose_model._build_graph()
del net_data
###################
# Shape CNN
###################
x2 = tf.image.resize_bilinear(x, tf.constant([224,224], dtype=tf.int32))
x2 = tf.cast(x2, 'float32')
x2 = tf.reshape(x2, [FLAGS.batch_size, 224, 224, 3])
# Image normalization
mean = tf.reshape(mean_image_shape, [1, 224, 224, 3])
mean = tf.cast(mean, 'float32')
x2 = x2 - mean
with tf.variable_scope('shapeCNN'):
net_shape = resnet101_shape({'input': x2}, trainable=True)
pool5 = net_shape.layers['pool5']
pool5 = tf.squeeze(pool5)
pool5 = tf.reshape(pool5, [FLAGS.batch_size,-1])
npzfile = np.load('./ResNet/ShapeNet_fc_weights.npz')
ini_weights_shape = npzfile['ini_weights_shape']
ini_biases_shape = npzfile['ini_biases_shape']
with tf.variable_scope('shapeCNN_fc1'):
fc1ws = tf.Variable(tf.reshape(ini_weights_shape, [2048,-1]), trainable=True, name='weights')
fc1bs = tf.Variable(tf.reshape(ini_biases_shape, [-1]), trainable=True, name='biases')
fc1ls = tf.nn.bias_add(tf.matmul(pool5, fc1ws), fc1bs)
###################
# Expression CNN
###################
with tf.variable_scope('exprCNN'):
net_expr = resnet101_expr({'input': x2}, trainable=True)
pool5 = net_expr.layers['pool5']
pool5 = tf.squeeze(pool5)
pool5 = tf.reshape(pool5, [FLAGS.batch_size,-1])
npzfile = np.load('./ResNet/ExpNet_fc_weights.npz')
ini_weights_expr = npzfile['ini_weights_expr']
ini_biases_expr = npzfile['ini_biases_expr']
with tf.variable_scope('exprCNN_fc1'):
fc1we = tf.Variable(tf.reshape(ini_weights_expr, [2048,29]), trainable=True, name='weights')
fc1be = tf.Variable(tf.reshape(ini_biases_expr, [29]), trainable=True, name='biases')
fc1le = tf.nn.bias_add(tf.matmul(pool5, fc1we), fc1be)
# Add ops to save and restore all the variables.
init_op = tf.global_variables_initializer()
saver_pose = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Spatial_Transformer'))
saver_ini_shape_net = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='shapeCNN'))
saver_ini_expr_net = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='exprCNN'))
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
sess.run(init_op)
# Load face pose net model from Chang et al.'ICCVW17
load_path = "./fpn_new_model/model_0_1.0_1.0_1e-07_1_16000.ckpt"
saver_pose.restore(sess, load_path)
# load 3dmm shape and texture model from Tran et al.' CVPR2017
load_path = "./Shape_Model/ini_ShapeTextureNet_model.ckpt"
saver_ini_shape_net.restore(sess, load_path)
# load our expression net model
load_path = "./Expression_Model/ini_exprNet_model.ckpt"
saver_ini_expr_net.restore(sess, load_path)
## Modifed Basel Face Model
BFM_path = './Shape_Model/BaselFaceModel_mod.mat'
model = scipy.io.loadmat(BFM_path,squeeze_me=True,struct_as_record=False)
model = model["BFM"]
faces = model.faces-1
print('> Loaded the Basel Face Model to write the 3D output!')
print('> Start to estimate Expression, Shape, and Pose!')
with open(output_proc, 'r') as csvfile:
csvreader = csv.reader(csvfile, delimiter=',')
for row in csvreader:
image_key = row[0]
image_file_path = row[1]
start = time.time()
print('> Process ' + image_file_path)
image = cv2.imread(image_file_path,1) # BGR
image = np.asarray(image)
# Fix the grey image
if len(image.shape) < 3:
image_r = np.reshape(image, (image.shape[0], image.shape[1], 1))
image = np.append(image_r, image_r, axis=2)
image = np.append(image, image_r, axis=2)
image = np.reshape(image, [1, FLAGS.image_size, FLAGS.image_size, 3])
(Shape_Texture, Expr, Pose) = sess.run([fc1ls, fc1le, pose_model.preds_unNormalized], feed_dict={x: image})
outFile = mesh_folder + '/' + image_key
Pose = np.reshape(Pose, [-1])
Shape_Texture = np.reshape(Shape_Texture, [-1])
Shape = Shape_Texture[0:99]
Shape = np.reshape(Shape, [-1])
Expr = np.reshape(Expr, [-1])
#########################################
### Save 3D shape information (.ply file)
#########################################
# Shape Only
#S,T = utils.projectBackBFM(model,Shape_Texture)
#utils.write_ply_textureless(outFile + '_ShapeOnly.ply', S, faces)
# Shape + Expression
#SE,TE = utils.projectBackBFM_withExpr(model, Shape_Texture, Expr)
#utils.write_ply_textureless(outFile + '_Shape_and_Expr.ply', SE, faces)
# Shape + Expression + Pose
SEP,TEP = utils.projectBackBFM_withEP(model, Shape_Texture, Expr, Pose)
utils.write_ply_textureless(outFile + '_Shape_Expr_Pose.ply', SEP, faces)
end = time.time()
print(end - start)
def extract_PSE_feats():
# Prepare data
data_dict = myparse.parse_input('./input.csv') # please see input.csv for the input format
print len(data_dict)
## Pre-processing the images
print '> preproc'
pu.preProcessImage(_tmpdir, data_dict, './', factor, _alexNetSize, output_proc)
# placeholders for the batches
x = tf.placeholder(tf.float32, [FLAGS.batch_size, FLAGS.image_size, FLAGS.image_size, 3])
###################
# Face Pose-Net
###################
net_data = np.load("./fpn_new_model/PAM_frontal_ALexNet.npy").item()
pose_labels = np.zeros([FLAGS.batch_size,6])
x1 = tf.image.resize_bilinear(x, tf.constant([227,227], dtype=tf.int32))
# Image normalization
x1 = x1 / 255. # from [0,255] to [0,1]
# subtract training mean
mean = tf.reshape(train_mean_vec, [1, 1, 1, 3])
mean = tf.cast(mean, 'float32')
x1 = x1 - mean
pose_model = Pose_model.Pose_Estimation(x1, pose_labels, 'valid', 0, 1, 1, 0.01, net_data, FLAGS.batch_size, mean_labels, std_labels)
pose_model._build_graph()
del net_data
###################
# Shape CNN
###################
x2 = tf.image.resize_bilinear(x, tf.constant([224,224], dtype=tf.int32))
x2 = tf.cast(x2, 'float32')
x2 = tf.reshape(x2, [FLAGS.batch_size, 224, 224, 3])
# Image normalization
mean = tf.reshape(mean_image_shape, [1, 224, 224, 3])
mean = tf.cast(mean, 'float32')
x2 = x2 - mean
with tf.variable_scope('shapeCNN'):
net_shape = resnet101_shape({'input': x2}, trainable=True)
pool5 = net_shape.layers['pool5']
pool5 = tf.squeeze(pool5)
pool5 = tf.reshape(pool5, [FLAGS.batch_size,-1])
npzfile = np.load('./ResNet/ShapeNet_fc_weights.npz')
ini_weights_shape = npzfile['ini_weights_shape']
ini_biases_shape = npzfile['ini_biases_shape']
with tf.variable_scope('shapeCNN_fc1'):
fc1ws = tf.Variable(tf.reshape(ini_weights_shape, [2048,-1]), trainable=True, name='weights')
fc1bs = tf.Variable(tf.reshape(ini_biases_shape, [-1]), trainable=True, name='biases')
fc1ls = tf.nn.bias_add(tf.matmul(pool5, fc1ws), fc1bs)
###################
# Expression CNN
###################
with tf.variable_scope('exprCNN'):
net_expr = resnet101_expr({'input': x2}, trainable=True)
pool5 = net_expr.layers['pool5']
pool5 = tf.squeeze(pool5)
pool5 = tf.reshape(pool5, [FLAGS.batch_size,-1])
npzfile = np.load('./ResNet/ExpNet_fc_weights.npz')
ini_weights_expr = npzfile['ini_weights_expr']
ini_biases_expr = npzfile['ini_biases_expr']
with tf.variable_scope('exprCNN_fc1'):
fc1we = tf.Variable(tf.reshape(ini_weights_expr, [2048,29]), trainable=True, name='weights')
fc1be = tf.Variable(tf.reshape(ini_biases_expr, [29]), trainable=True, name='biases')
fc1le = tf.nn.bias_add(tf.matmul(pool5, fc1we), fc1be)
# Add ops to save and restore all the variables.
init_op = tf.global_variables_initializer()
saver_pose = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Spatial_Transformer'))
saver_ini_shape_net = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='shapeCNN'))
saver_ini_expr_net = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='exprCNN'))
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
sess.run(init_op)
# Load face pose net model from Chang et al.'ICCVW17
load_path = "./fpn_new_model/model_0_1.0_1.0_1e-07_1_16000.ckpt"
saver_pose.restore(sess, load_path)
# load 3dmm shape and texture model from Tran et al.' CVPR2017
load_path = "./Shape_Model/ini_ShapeTextureNet_model.ckpt"
saver_ini_shape_net.restore(sess, load_path)
# load our expression net model
load_path = "./Expression_Model/ini_exprNet_model.ckpt"
saver_ini_expr_net.restore(sess, load_path)
## Modifed Basel Face Model
BFM_path = './Shape_Model/BaselFaceModel_mod.mat'
model = scipy.io.loadmat(BFM_path,squeeze_me=True,struct_as_record=False)
model = model["BFM"]
faces = model.faces-1
print '> Loaded the Basel Face Model to write the 3D output!'
print '> Start to estimate Expression, Shape, and Pose!'
with open(output_proc, 'rb') as csvfile:
csvreader = csv.reader(csvfile, delimiter=',')
for row in csvreader:
image_key = row[0]
image_file_path = row[1]
print '> Process ' + image_file_path
image = cv2.imread(image_file_path,1) # BGR
image = np.asarray(image)
# Fix the grey image
if len(image.shape) < 3:
image_r = np.reshape(image, (image.shape[0], image.shape[1], 1))
image = np.append(image_r, image_r, axis=2)
image = np.append(image, image_r, axis=2)
image = np.reshape(image, [1, FLAGS.image_size, FLAGS.image_size, 3])
(Shape_Texture, Expr, Pose) = sess.run([fc1ls, fc1le, pose_model.preds_unNormalized], feed_dict={x: image})
outFile = mesh_folder + '/' + image_key
Pose = np.reshape(Pose, [-1])
Shape_Texture = np.reshape(Shape_Texture, [-1])
Shape = Shape_Texture[0:99]
Shape = np.reshape(Shape, [-1])
Expr = np.reshape(Expr, [-1])
#########################################
### Save 3D shape information (.ply file)
#########################################
# Shape Only
#S,T = utils.projectBackBFM(model,Shape_Texture)
#utils.write_ply_textureless(outFile + '_ShapeOnly.ply', S, faces)
# Shape + Expression
#SE,TE = utils.projectBackBFM_withExpr(model, Shape_Texture, Expr)
#utils.write_ply_textureless(outFile + '_Shape_and_Expr.ply', SE, faces)
# Shape + Expression + Pose
SEP,TEP = utils.projectBackBFM_withEP(model, Shape_Texture, Expr, Pose)
utils.write_ply_textureless(outFile + '_Shape_Expr_Pose.ply', SEP, faces)