def extract_PSE_feats():
# Prepare data
data_dict = myparse.parse_input(inputlist) # please see input.csv for the input format
print(len(data_dict))
## Pre-processing the images
print('> preproc')
pu.preProcessImage(_tmpdir, data_dict, './', factor, _alexNetSize, output_proc)
# placeholders for the batches
x = tf.placeholder(tf.float32, [FLAGS.batch_size, FLAGS.image_size, FLAGS.image_size, 3])
###################
# Face Pose-Net
###################
net_data = np.load("./fpn_new_model/PAM_frontal_ALexNet_py3.npy").item()
pose_labels = np.zeros([FLAGS.batch_size,6])
x1 = tf.image.resize_bilinear(x, tf.constant([227,227], dtype=tf.int32))
# Image normalization
x1 = x1 / 255. # from [0,255] to [0,1]
# subtract training mean
mean = tf.reshape(train_mean_vec, [1, 1, 1, 3])
mean = tf.cast(mean, 'float32')
x1 = x1 - mean
pose_model = Pose_model.Pose_Estimation(x1, pose_labels, 'valid', 0, 1, 1, 0.01, net_data, FLAGS.batch_size, mean_labels, std_labels)
pose_model._build_graph()
del net_data
###################
# Shape CNN
###################
x2 = tf.image.resize_bilinear(x, tf.constant([224,224], dtype=tf.int32))
x2 = tf.cast(x2, 'float32')
x2 = tf.reshape(x2, [FLAGS.batch_size, 224, 224, 3])
# Image normalization
mean = tf.reshape(mean_image_shape, [1, 224, 224, 3])
mean = tf.cast(mean, 'float32')
x2 = x2 - mean
with tf.variable_scope('shapeCNN'):
net_shape = resnet101_shape({'input': x2}, trainable=True)
pool5 = net_shape.layers['pool5']
pool5 = tf.squeeze(pool5)
pool5 = tf.reshape(pool5, [FLAGS.batch_size,-1])
npzfile = np.load('./ResNet/ShapeNet_fc_weights.npz')
ini_weights_shape = npzfile['ini_weights_shape']
ini_biases_shape = npzfile['ini_biases_shape']
with tf.variable_scope('shapeCNN_fc1'):
fc1ws = tf.Variable(tf.reshape(ini_weights_shape, [2048,-1]), trainable=True, name='weights')
fc1bs = tf.Variable(tf.reshape(ini_biases_shape, [-1]), trainable=True, name='biases')
fc1ls = tf.nn.bias_add(tf.matmul(pool5, fc1ws), fc1bs)
###################
# Expression CNN
###################
with tf.variable_scope('exprCNN'):
net_expr = resnet101_expr({'input': x2}, trainable=True)
pool5 = net_expr.layers['pool5']
pool5 = tf.squeeze(pool5)
pool5 = tf.reshape(pool5, [FLAGS.batch_size,-1])
npzfile = np.load('./ResNet/ExpNet_fc_weights.npz')
ini_weights_expr = npzfile['ini_weights_expr']
ini_biases_expr = npzfile['ini_biases_expr']
with tf.variable_scope('exprCNN_fc1'):
fc1we = tf.Variable(tf.reshape(ini_weights_expr, [2048,29]), trainable=True, name='weights')
fc1be = tf.Variable(tf.reshape(ini_biases_expr, [29]), trainable=True, name='biases')
fc1le = tf.nn.bias_add(tf.matmul(pool5, fc1we), fc1be)
# Add ops to save and restore all the variables.
init_op = tf.global_variables_initializer()
saver_pose = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Spatial_Transformer'))
saver_ini_shape_net = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='shapeCNN'))
saver_ini_expr_net = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='exprCNN'))
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
sess.run(init_op)
# Load face pose net model from Chang et al.'ICCVW17
load_path = "./fpn_new_model/model_0_1.0_1.0_1e-07_1_16000.ckpt"
saver_pose.restore(sess, load_path)
# load 3dmm shape and texture model from Tran et al.' CVPR2017
load_path = "./Shape_Model/ini_ShapeTextureNet_model.ckpt"
saver_ini_shape_net.restore(sess, load_path)
# load our expression net model
load_path = "./Expression_Model/ini_exprNet_model.ckpt"
saver_ini_expr_net.restore(sess, load_path)
## Modifed Basel Face Model
BFM_path = './Shape_Model/BaselFaceModel_mod.mat'
model = scipy.io.loadmat(BFM_path,squeeze_me=True,struct_as_record=False)
model = model["BFM"]
faces = model.faces-1
print('> Loaded the Basel Face Model to write the 3D output!')
print('> Start to estimate Expression, Shape, and Pose!')
with open(output_proc, 'r') as csvfile:
csvreader = csv.reader(csvfile, delimiter=',')
for row in csvreader:
image_key = row[0]
image_file_path = row[1]
start = time.time()
print('> Process ' + image_file_path)
image = cv2.imread(image_file_path,1) # BGR
image = np.asarray(image)
# Fix the grey image
if len(image.shape) < 3:
image_r = np.reshape(image, (image.shape[0], image.shape[1], 1))
image = np.append(image_r, image_r, axis=2)
image = np.append(image, image_r, axis=2)
image = np.reshape(image, [1, FLAGS.image_size, FLAGS.image_size, 3])
(Shape_Texture, Expr, Pose) = sess.run([fc1ls, fc1le, pose_model.preds_unNormalized], feed_dict={x: image})
outFile = mesh_folder + '/' + image_key
Pose = np.reshape(Pose, [-1])
Shape_Texture = np.reshape(Shape_Texture, [-1])
Shape = Shape_Texture[0:99]
Shape = np.reshape(Shape, [-1])
Expr = np.reshape(Expr, [-1])
#########################################
### Save 3D shape information (.ply file)
#########################################
# Shape Only
#S,T = utils.projectBackBFM(model,Shape_Texture)
#utils.write_ply_textureless(outFile + '_ShapeOnly.ply', S, faces)
# Shape + Expression
#SE,TE = utils.projectBackBFM_withExpr(model, Shape_Texture, Expr)
#utils.write_ply_textureless(outFile + '_Shape_and_Expr.ply', SE, faces)
# Shape + Expression + Pose
SEP,TEP = utils.projectBackBFM_withEP(model, Shape_Texture, Expr, Pose)
utils.write_ply_textureless(outFile + '_Shape_Expr_Pose.ply', SEP, faces)
end = time.time()
print(end - start)
def extract_PSE_feats():
# Prepare data
data_dict = myparse.parse_input('./input.csv') # please see input.csv for the input format
print len(data_dict)
## Pre-processing the images
print '> preproc'
pu.preProcessImage(_tmpdir, data_dict, './', factor, _alexNetSize, output_proc)
# placeholders for the batches
x = tf.placeholder(tf.float32, [FLAGS.batch_size, FLAGS.image_size, FLAGS.image_size, 3])
###################
# Face Pose-Net
###################
net_data = np.load("./fpn_new_model/PAM_frontal_ALexNet.npy").item()
pose_labels = np.zeros([FLAGS.batch_size,6])
x1 = tf.image.resize_bilinear(x, tf.constant([227,227], dtype=tf.int32))
# Image normalization
x1 = x1 / 255. # from [0,255] to [0,1]
# subtract training mean
mean = tf.reshape(train_mean_vec, [1, 1, 1, 3])
mean = tf.cast(mean, 'float32')
x1 = x1 - mean
pose_model = Pose_model.Pose_Estimation(x1, pose_labels, 'valid', 0, 1, 1, 0.01, net_data, FLAGS.batch_size, mean_labels, std_labels)
pose_model._build_graph()
del net_data
###################
# Shape CNN
###################
x2 = tf.image.resize_bilinear(x, tf.constant([224,224], dtype=tf.int32))
x2 = tf.cast(x2, 'float32')
x2 = tf.reshape(x2, [FLAGS.batch_size, 224, 224, 3])
# Image normalization
mean = tf.reshape(mean_image_shape, [1, 224, 224, 3])
mean = tf.cast(mean, 'float32')
x2 = x2 - mean
with tf.variable_scope('shapeCNN'):
net_shape = resnet101_shape({'input': x2}, trainable=True)
pool5 = net_shape.layers['pool5']
pool5 = tf.squeeze(pool5)
pool5 = tf.reshape(pool5, [FLAGS.batch_size,-1])
npzfile = np.load('./ResNet/ShapeNet_fc_weights.npz')
ini_weights_shape = npzfile['ini_weights_shape']
ini_biases_shape = npzfile['ini_biases_shape']
with tf.variable_scope('shapeCNN_fc1'):
fc1ws = tf.Variable(tf.reshape(ini_weights_shape, [2048,-1]), trainable=True, name='weights')
fc1bs = tf.Variable(tf.reshape(ini_biases_shape, [-1]), trainable=True, name='biases')
fc1ls = tf.nn.bias_add(tf.matmul(pool5, fc1ws), fc1bs)
###################
# Expression CNN
###################
with tf.variable_scope('exprCNN'):
net_expr = resnet101_expr({'input': x2}, trainable=True)
pool5 = net_expr.layers['pool5']
pool5 = tf.squeeze(pool5)
pool5 = tf.reshape(pool5, [FLAGS.batch_size,-1])
npzfile = np.load('./ResNet/ExpNet_fc_weights.npz')
ini_weights_expr = npzfile['ini_weights_expr']
ini_biases_expr = npzfile['ini_biases_expr']
with tf.variable_scope('exprCNN_fc1'):
fc1we = tf.Variable(tf.reshape(ini_weights_expr, [2048,29]), trainable=True, name='weights')
fc1be = tf.Variable(tf.reshape(ini_biases_expr, [29]), trainable=True, name='biases')
fc1le = tf.nn.bias_add(tf.matmul(pool5, fc1we), fc1be)
# Add ops to save and restore all the variables.
init_op = tf.global_variables_initializer()
saver_pose = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Spatial_Transformer'))
saver_ini_shape_net = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='shapeCNN'))
saver_ini_expr_net = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='exprCNN'))
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
sess.run(init_op)
# Load face pose net model from Chang et al.'ICCVW17
load_path = "./fpn_new_model/model_0_1.0_1.0_1e-07_1_16000.ckpt"
saver_pose.restore(sess, load_path)
# load 3dmm shape and texture model from Tran et al.' CVPR2017
load_path = "./Shape_Model/ini_ShapeTextureNet_model.ckpt"
saver_ini_shape_net.restore(sess, load_path)
# load our expression net model
load_path = "./Expression_Model/ini_exprNet_model.ckpt"
saver_ini_expr_net.restore(sess, load_path)
## Modifed Basel Face Model
BFM_path = './Shape_Model/BaselFaceModel_mod.mat'
model = scipy.io.loadmat(BFM_path,squeeze_me=True,struct_as_record=False)
model = model["BFM"]
faces = model.faces-1
print '> Loaded the Basel Face Model to write the 3D output!'
print '> Start to estimate Expression, Shape, and Pose!'
with open(output_proc, 'rb') as csvfile:
csvreader = csv.reader(csvfile, delimiter=',')
for row in csvreader:
image_key = row[0]
image_file_path = row[1]
print '> Process ' + image_file_path
image = cv2.imread(image_file_path,1) # BGR
image = np.asarray(image)
# Fix the grey image
if len(image.shape) < 3:
image_r = np.reshape(image, (image.shape[0], image.shape[1], 1))
image = np.append(image_r, image_r, axis=2)
image = np.append(image, image_r, axis=2)
image = np.reshape(image, [1, FLAGS.image_size, FLAGS.image_size, 3])
(Shape_Texture, Expr, Pose) = sess.run([fc1ls, fc1le, pose_model.preds_unNormalized], feed_dict={x: image})
outFile = mesh_folder + '/' + image_key
Pose = np.reshape(Pose, [-1])
Shape_Texture = np.reshape(Shape_Texture, [-1])
Shape = Shape_Texture[0:99]
Shape = np.reshape(Shape, [-1])
Expr = np.reshape(Expr, [-1])
#########################################
### Save 3D shape information (.ply file)
#########################################
# Shape Only
#S,T = utils.projectBackBFM(model,Shape_Texture)
#utils.write_ply_textureless(outFile + '_ShapeOnly.ply', S, faces)
# Shape + Expression
#SE,TE = utils.projectBackBFM_withExpr(model, Shape_Texture, Expr)
#utils.write_ply_textureless(outFile + '_Shape_and_Expr.ply', SE, faces)
# Shape + Expression + Pose
SEP,TEP = utils.projectBackBFM_withEP(model, Shape_Texture, Expr, Pose)
utils.write_ply_textureless(outFile + '_Shape_Expr_Pose.ply', SEP, faces)
def extract_PSE_feats(self, x):
'''
:param x: x format is RGB and is value range is [-1,1].
:return: fc1ls: shape, fc1le: expression, pose_model.preds_unNormalized: pose
'''
x = tf.image.resize_images(x, [227, 227])
# x is RGB and is value range is [-1,1].
# first we need to change RGB to BGR;
batch_, height_, width_, nc = x.get_shape().as_list()
R = tf.reshape(x[:, :, :, 0], [batch_, height_, width_, 1])
G = tf.reshape(x[:, :, :, 1], [batch_, height_, width_, 1])
B = tf.reshape(x[:, :, :, 2], [batch_, height_, width_, 1])
x = tf.concat([B, G, R], axis=3)
# second change range [-1,1] to [0,255]
x = (x + 1.0) * 127.5
###################
# Face Pose-Net
###################
try:
net_data = np.load(self.cfg.PAM_frontal_ALexNet_file_path,
encoding="latin1").item()
pose_labels = np.zeros([self.cfg.batch_size, 6])
print('Load ' + self.cfg.PAM_frontal_ALexNet_file_path +
' successful....')
except:
raise Exception('Load ' + self.cfg.PAM_frontal_ALexNet_file_path +
' failed....')
x1 = tf.image.resize_bilinear(x, tf.constant([227, 227],
dtype=tf.int32))
# Image normalization
x1 = x1 / 255. # from [0,255] to [0,1]
# subtract training mean
mean = tf.reshape(self.train_mean_vec, [1, 1, 1, 3])
mean = tf.cast(mean, 'float32')
x1 = x1 - mean
pose_model = Pose_model.Pose_Estimation(x1, pose_labels, 'valid', 0, 1,
1, 0.01, net_data,
self.cfg.batch_size,
self.mean_labels,
self.std_labels)
pose_model._build_graph()
self.pose = pose_model.preds_unNormalized
del net_data
###################
# Shape CNN
###################
x2 = tf.image.resize_bilinear(x, tf.constant([224, 224],
dtype=tf.int32))
x2 = tf.cast(x2, 'float32')
x2 = tf.reshape(x2, [self.cfg.batch_size, 224, 224, 3])
# Image normalization
mean = tf.reshape(self.mean_image_shape, [1, 224, 224, 3])
mean = tf.cast(mean, 'float32')
x2 = x2 - mean
with tf.variable_scope('shapeCNN'):
net_shape = resnet101_shape({'input': x2}, trainable=True)
pool5 = net_shape.layers['pool5']
pool5 = tf.squeeze(pool5)
pool5 = tf.reshape(pool5, [self.cfg.batch_size, -1])
try:
npzfile = np.load(self.cfg.ShapeNet_fc_weights_file_path)
print('Load ' + self.cfg.ShapeNet_fc_weights_file_path +
' successful....')
except:
raise Exception('Load ' +
self.cfg.ShapeNet_fc_weights_file_path +
' failed....')
ini_weights_shape = npzfile['ini_weights_shape']
ini_biases_shape = npzfile['ini_biases_shape']
with tf.variable_scope('shapeCNN_fc1'):
fc1ws = tf.Variable(tf.reshape(ini_weights_shape, [2048, -1]),
trainable=True,
name='weights')
fc1bs = tf.Variable(tf.reshape(ini_biases_shape, [-1]),
trainable=True,
name='biases')
self.fc1ls = tf.nn.bias_add(tf.matmul(pool5, fc1ws), fc1bs)
###################
# Expression CNN
###################
with tf.variable_scope('exprCNN'):
net_expr = resnet101_expr({'input': x2}, trainable=True)
pool5 = net_expr.layers['pool5']
pool5 = tf.squeeze(pool5)
pool5 = tf.reshape(pool5, [self.cfg.batch_size, -1])
try:
npzfile = np.load(self.cfg.ExpNet_fc_weights_file_path)
ini_weights_expr = npzfile['ini_weights_expr']
ini_biases_expr = npzfile['ini_biases_expr']
print('Load ' + self.cfg.ExpNet_fc_weights_file_path +
' successful....')
except:
raise Exception('Load ' +
self.cfg.ExpNet_fc_weights_file_path +
' failed....')
# time.sleep(30)
with tf.variable_scope('exprCNN_fc1'):
fc1we = tf.Variable(tf.reshape(ini_weights_expr, [2048, 29]),
trainable=True,
name='weights')
fc1be = tf.Variable(tf.reshape(ini_biases_expr, [29]),
trainable=True,
name='biases')
self.fc1le = tf.nn.bias_add(tf.matmul(pool5, fc1we), fc1be)
def extract_3dmm_ProjMat():
########################################
# Load train image mean, train label mean and std
########################################
# labels stats on 300W-LP
train_label_mean = np.load('./train_stats/train_label_mean_ProjMat.npy')
train_label_std = np.load('./train_stats/train_label_std_ProjMat.npy')
ProjMat_label_mean = train_label_mean[-12:-1]
ProjMat_label_std = train_label_std[-12:-1]
# Get training image mean from Anh's ShapeNet (CVPR2017)
mean_image_shape = np.load(
'./train_stats/3DMM_shape_mean.npy') # 3 x 224 x 224
train_image_mean = np.transpose(mean_image_shape,
[1, 2, 0]) # 224 x 224 x 3, [0,255]
########################################
# Build CNN graph
########################################
# placeholders for the batches
x_img = tf.placeholder(tf.float32,
[None, FLAGS.image_size, FLAGS.image_size, 3])
# Resize Image
x2 = tf.image.resize_bilinear(x_img, tf.constant([224, 224],
dtype=tf.int32))
x2 = tf.cast(x2, 'float32')
x2 = tf.reshape(x2, [-1, 224, 224, 3])
# Image normalization
mean = tf.reshape(train_image_mean, [1, 224, 224, 3])
mean = tf.cast(mean, 'float32')
x2 = x2 - mean
########################################
# New-FPN with ResNet structure
########################################
with tf.variable_scope('shapeCNN'):
net_shape = resnet101_shape(
{'input': x2}, trainable=True) # False: Freeze the ResNet Layers
pool5 = net_shape.layers['pool5']
pool5 = tf.squeeze(pool5)
pool5 = tf.reshape(pool5, [1, 2048])
print pool5.get_shape() # batch_size x 2048
with tf.variable_scope('Pose'):
with tf.variable_scope('fc1'):
fc1W = tf.Variable(tf.random_normal(tf.stack(
[pool5.get_shape()[1].value, n_hidden1]),
mean=0.0,
stddev=0.01),
trainable=True,
name='W')
fc1b = tf.Variable(tf.zeros([n_hidden1]),
trainable=True,
name='baises')
fc1 = tf.nn.relu_layer(tf.reshape(
pool5, [-1, int(np.prod(pool5.get_shape()[1:]))]),
fc1W,
fc1b,
name='fc1')
print "\nfc1 shape:"
print fc1.get_shape(), fc1W.get_shape(), fc1b.get_shape(
) # (batch_size, 4096) (2048, 4096) (4096,)
if ifdropout == 1:
fc1 = tf.nn.dropout(fc1, prob, name='fc1_dropout')
with tf.variable_scope('fc2'):
fc2W = tf.Variable(tf.random_normal([n_hidden1, n_hidden2],
mean=0.0,
stddev=0.01),
trainable=True,
name='W')
fc2b = tf.Variable(tf.zeros([n_hidden2]),
trainable=True,
name='baises')
fc2 = tf.nn.relu_layer(fc1, fc2W, fc2b, name='fc2')
print fc2.get_shape(), fc2W.get_shape(), fc2b.get_shape(
) # (batch_size, 29 (2048, 2048) (2048,)
if ifdropout == 1:
fc2 = tf.nn.dropout(fc2, prob, name='fc2_dropout')
with tf.variable_scope('fc3'):
# Move everything into depth so we can perform a single matrix multiplication.
fc2 = tf.reshape(fc2, [FLAGS.batch_size, -1])
dim = fc2.get_shape()[1].value
print "\nfc2 dim:"
print fc2.get_shape(), dim
fc3W = tf.Variable(tf.random_normal(tf.stack([dim, 11]),
mean=0.0,
stddev=0.01),
trainable=True,
name='W')
fc3b = tf.Variable(tf.zeros([11]), trainable=True, name='baises')
#print "*** label shape: " + str(len(train_label_mean))
ProjMat_preds_ZNorm = tf.nn.xw_plus_b(fc2, fc3W, fc3b)
print "\nfc3 shape:"
print ProjMat_preds_ZNorm.get_shape(), fc3W.get_shape(
), fc3b.get_shape()
label_mean = tf.cast(tf.reshape(ProjMat_label_mean, [1, -1]),
'float32')
label_std = tf.cast(tf.reshape(ProjMat_label_std, [1, -1]),
'float32')
ProjMat_preds = ProjMat_preds_ZNorm * (
label_std + 0.000000000000000001) + label_mean
ProjMat_preds = tf.concat(
[ProjMat_preds, tf.zeros([FLAGS.batch_size, 1])], 1)
########################################
# Start extracting 3dmm pose
########################################
init_op = tf.global_variables_initializer()
saver = tf.train.Saver(
var_list=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES))
saver_ini_shape_net = tf.train.Saver(var_list=tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES, scope='shapeCNN'))
saver_shapeCNN = tf.train.Saver(var_list=tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope='shapeCNN'))
saver_Pose = tf.train.Saver(var_list=tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope='Pose'))
config = tf.ConfigProto(
allow_soft_placement=True) #, log_device_placement=True)
#config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
start_time = time.time()
load_path = "./models/ini_shapeNet_model_L7L_trainable.ckpt"
saver_ini_shape_net.restore(sess, load_path)
load_path = "./models/model_0.0001_1_18_0.0_2048_4096.ckpt"
saver_shapeCNN.restore(sess, load_path)
load_path = "./models/model_iniLR_0.001_wProjMat_1.0_wLmks_10.0_wd_0.0_do_1_122.ckpt"
saver_Pose.restore(sess, load_path)
load_model_time = time.time() - start_time
print("Model restored: " + str(load_model_time))
with open(input_sample_list_path, 'r') as fin:
for line in fin:
curr_line = line.strip().split(',')
image_path = curr_line[0]
bbox = np.array([
float(curr_line[1]),
float(curr_line[2]),
float(curr_line[3]),
float(curr_line[4])
]) # [lt_x, lt_y, w, h]
image_key = image_path.split('/')[-1][:-4]
image = cv2.imread(image_path, 1) # BGR
image = np.asarray(image)
# Fix the grey image
if len(image.shape) < 3:
image_r = np.reshape(image,
(image.shape[0], image.shape[1], 1))
image = np.append(image_r, image_r, axis=2)
image = np.append(image, image_r, axis=2)
# Crop and expand (25%) the image based on the tight bbox (from the face detector or detected lmks)
factor = [1.9255, 2.2591, 1.9423, 1.6087]
img_new = pu.preProcessImage_v2(image.copy(), bbox.copy(),
factor, _resNetSize, 1)
image_array = np.reshape(img_new,
[1, _resNetSize, _resNetSize, 3])
#print image_array
(params_ProjMat, pool5_feats) = sess.run(
[ProjMat_preds, pool5], feed_dict={x_img: image_array}
) # [scale, pitch, yaw, roll, translation_x, translation_y]
params_ProjMat = params_ProjMat[0]
#print params_ProjMat, pool5_feats
# save the predicted pose
with open(FLAGS.save_output_path + '/' + image_key + '.txt',
'w') as fout:
for pp in params_ProjMat:
fout.write(str(pp) + '\n')
# Convert the 6DoF predicted pose to 3x4 projection matrix (weak-perspective projection)
# Load BFM model
shape_mat = sio.loadmat('./BFM/Model_Shape.mat')
mu_shape = shape_mat['mu_shape'].astype('float32')
expr_mat = sio.loadmat('./BFM/Model_Exp.mat')
mu_exp = expr_mat['mu_exp'].astype('float32')
mu = mu_shape + mu_exp
len_mu = len(mu)
mu = np.reshape(mu, [-1, 1])
keypoints = np.reshape(shape_mat['keypoints'],
[-1]) - 1 # -1 for python index
keypoints = keypoints.astype('int32')
vertex = np.reshape(mu, [len_mu / 3, 3]) # # of vertices x 3
# mean shape
mesh = vertex.T # 3 x # of vertices
mesh_1 = np.concatenate([mesh, np.ones([1, len_mu / 3])],
axis=0) # 4 x # of vertices
# Get projection matrix from 6DoF pose
ProjMat = np.reshape(params_ProjMat, [4, 3])
ProjMat = ProjMat.T
# Get predicted shape
#print ProjMat, ProjMat.shape
#print mesh_1, mesh_1.shape
pred_shape = np.matmul(ProjMat, mesh_1) # 3 x # of vertices
pred_shape = pred_shape.T # # of vertices x 3
pred_shape_x = np.reshape(pred_shape[:, 0], [len_mu / 3, 1])
pred_shape_z = np.reshape(pred_shape[:, 2], [len_mu / 3, 1])
pred_shape_y = 224 + 1 - pred_shape[:, 1]
pred_shape_y = np.reshape(pred_shape_y, [len_mu / 3, 1])
pred_shape = np.concatenate(
[pred_shape_x, pred_shape_y, pred_shape_z], 1)
# Convert shape and lmks back to the original image scale
_, bbox_new, _, _, old_h, old_w, _ = pu.resize_crop_rescaleCASIA(
image.copy(), bbox.copy(), pred_shape.copy(), factor)
pred_shape[:, 0] = pred_shape[:, 0] * old_w / 224.
pred_shape[:, 1] = pred_shape[:, 1] * old_h / 224.
pred_shape[:, 0] = pred_shape[:, 0] + bbox_new[0]
pred_shape[:, 1] = pred_shape[:, 1] + bbox_new[1]
# Get predicted lmks
pred_lmks = pred_shape[keypoints]
sio.savemat(FLAGS.save_output_path + '/' + image_key + '.mat',
{
'shape_3D': pred_shape,
'lmks_3D': pred_lmks
})
# Obtain pose from ProjMat
scale, R, t3d = pu.P2sRt(
ProjMat) # decompose affine matrix to s, R, t
pose = pu.matrix2angle(R) # yaw, pitch, roll
# print scale, pitch, yaw , roll, translation_x, translation_y
print scale, pose[1], pose[0], pose[2], t3d[0], t3d[1]