def extract_PSE_feats(self, x): ''' :param x: x format is RGB and is value range is [-1,1]. :return: fc1ls: shape, fc1le: expression, pose_model.preds_unNormalized: pose ''' x = tf.image.resize_images(x, [227, 227]) # x is RGB and is value range is [-1,1]. # first we need to change RGB to BGR; batch_, height_, width_, nc = x.get_shape().as_list() R = tf.reshape(x[:, :, :, 0], [batch_, height_, width_, 1]) G = tf.reshape(x[:, :, :, 1], [batch_, height_, width_, 1]) B = tf.reshape(x[:, :, :, 2], [batch_, height_, width_, 1]) x = tf.concat([B, G, R], axis=3) # second change range [-1,1] to [0,255] x = (x + 1.0) * 127.5 ################### # Face Pose-Net ################### try: net_data = np.load(self.cfg.PAM_frontal_ALexNet_file_path, encoding="latin1").item() pose_labels = np.zeros([self.cfg.batch_size, 6]) print('Load ' + self.cfg.PAM_frontal_ALexNet_file_path + ' successful....') except: raise Exception('Load ' + self.cfg.PAM_frontal_ALexNet_file_path + ' failed....') x1 = tf.image.resize_bilinear(x, tf.constant([227, 227], dtype=tf.int32)) # Image normalization x1 = x1 / 255. # from [0,255] to [0,1] # subtract training mean mean = tf.reshape(self.train_mean_vec, [1, 1, 1, 3]) mean = tf.cast(mean, 'float32') x1 = x1 - mean pose_model = Pose_model.Pose_Estimation(x1, pose_labels, 'valid', 0, 1, 1, 0.01, net_data, self.cfg.batch_size, self.mean_labels, self.std_labels) pose_model._build_graph() self.pose = pose_model.preds_unNormalized del net_data ################### # Shape CNN ################### x2 = tf.image.resize_bilinear(x, tf.constant([224, 224], dtype=tf.int32)) x2 = tf.cast(x2, 'float32') x2 = tf.reshape(x2, [self.cfg.batch_size, 224, 224, 3]) # Image normalization mean = tf.reshape(self.mean_image_shape, [1, 224, 224, 3]) mean = tf.cast(mean, 'float32') x2 = x2 - mean with tf.variable_scope('shapeCNN'): net_shape = resnet101_shape({'input': x2}, trainable=True) pool5 = net_shape.layers['pool5'] pool5 = tf.squeeze(pool5) pool5 = tf.reshape(pool5, [self.cfg.batch_size, -1]) try: npzfile = np.load(self.cfg.ShapeNet_fc_weights_file_path) print('Load ' + self.cfg.ShapeNet_fc_weights_file_path + ' successful....') except: raise Exception('Load ' + self.cfg.ShapeNet_fc_weights_file_path + ' failed....') ini_weights_shape = npzfile['ini_weights_shape'] ini_biases_shape = npzfile['ini_biases_shape'] with tf.variable_scope('shapeCNN_fc1'): fc1ws = tf.Variable(tf.reshape(ini_weights_shape, [2048, -1]), trainable=True, name='weights') fc1bs = tf.Variable(tf.reshape(ini_biases_shape, [-1]), trainable=True, name='biases') self.fc1ls = tf.nn.bias_add(tf.matmul(pool5, fc1ws), fc1bs) ################### # Expression CNN ################### with tf.variable_scope('exprCNN'): net_expr = resnet101_expr({'input': x2}, trainable=True) pool5 = net_expr.layers['pool5'] pool5 = tf.squeeze(pool5) pool5 = tf.reshape(pool5, [self.cfg.batch_size, -1]) try: npzfile = np.load(self.cfg.ExpNet_fc_weights_file_path) ini_weights_expr = npzfile['ini_weights_expr'] ini_biases_expr = npzfile['ini_biases_expr'] print('Load ' + self.cfg.ExpNet_fc_weights_file_path + ' successful....') except: raise Exception('Load ' + self.cfg.ExpNet_fc_weights_file_path + ' failed....') # time.sleep(30) with tf.variable_scope('exprCNN_fc1'): fc1we = tf.Variable(tf.reshape(ini_weights_expr, [2048, 29]), trainable=True, name='weights') fc1be = tf.Variable(tf.reshape(ini_biases_expr, [29]), trainable=True, name='biases') self.fc1le = tf.nn.bias_add(tf.matmul(pool5, fc1we), fc1be)
def extract_PSE_feats(): # Prepare data data_dict = myparse.parse_input(inputlist) # please see input.csv for the input format print(len(data_dict)) ## Pre-processing the images print('> preproc') pu.preProcessImage(_tmpdir, data_dict, './', factor, _alexNetSize, output_proc) # placeholders for the batches x = tf.placeholder(tf.float32, [FLAGS.batch_size, FLAGS.image_size, FLAGS.image_size, 3]) ################### # Face Pose-Net ################### net_data = np.load("./fpn_new_model/PAM_frontal_ALexNet_py3.npy").item() pose_labels = np.zeros([FLAGS.batch_size,6]) x1 = tf.image.resize_bilinear(x, tf.constant([227,227], dtype=tf.int32)) # Image normalization x1 = x1 / 255. # from [0,255] to [0,1] # subtract training mean mean = tf.reshape(train_mean_vec, [1, 1, 1, 3]) mean = tf.cast(mean, 'float32') x1 = x1 - mean pose_model = Pose_model.Pose_Estimation(x1, pose_labels, 'valid', 0, 1, 1, 0.01, net_data, FLAGS.batch_size, mean_labels, std_labels) pose_model._build_graph() del net_data ################### # Shape CNN ################### x2 = tf.image.resize_bilinear(x, tf.constant([224,224], dtype=tf.int32)) x2 = tf.cast(x2, 'float32') x2 = tf.reshape(x2, [FLAGS.batch_size, 224, 224, 3]) # Image normalization mean = tf.reshape(mean_image_shape, [1, 224, 224, 3]) mean = tf.cast(mean, 'float32') x2 = x2 - mean with tf.variable_scope('shapeCNN'): net_shape = resnet101_shape({'input': x2}, trainable=True) pool5 = net_shape.layers['pool5'] pool5 = tf.squeeze(pool5) pool5 = tf.reshape(pool5, [FLAGS.batch_size,-1]) npzfile = np.load('./ResNet/ShapeNet_fc_weights.npz') ini_weights_shape = npzfile['ini_weights_shape'] ini_biases_shape = npzfile['ini_biases_shape'] with tf.variable_scope('shapeCNN_fc1'): fc1ws = tf.Variable(tf.reshape(ini_weights_shape, [2048,-1]), trainable=True, name='weights') fc1bs = tf.Variable(tf.reshape(ini_biases_shape, [-1]), trainable=True, name='biases') fc1ls = tf.nn.bias_add(tf.matmul(pool5, fc1ws), fc1bs) ################### # Expression CNN ################### with tf.variable_scope('exprCNN'): net_expr = resnet101_expr({'input': x2}, trainable=True) pool5 = net_expr.layers['pool5'] pool5 = tf.squeeze(pool5) pool5 = tf.reshape(pool5, [FLAGS.batch_size,-1]) npzfile = np.load('./ResNet/ExpNet_fc_weights.npz') ini_weights_expr = npzfile['ini_weights_expr'] ini_biases_expr = npzfile['ini_biases_expr'] with tf.variable_scope('exprCNN_fc1'): fc1we = tf.Variable(tf.reshape(ini_weights_expr, [2048,29]), trainable=True, name='weights') fc1be = tf.Variable(tf.reshape(ini_biases_expr, [29]), trainable=True, name='biases') fc1le = tf.nn.bias_add(tf.matmul(pool5, fc1we), fc1be) # Add ops to save and restore all the variables. init_op = tf.global_variables_initializer() saver_pose = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Spatial_Transformer')) saver_ini_shape_net = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='shapeCNN')) saver_ini_expr_net = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='exprCNN')) with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: sess.run(init_op) # Load face pose net model from Chang et al.'ICCVW17 load_path = "./fpn_new_model/model_0_1.0_1.0_1e-07_1_16000.ckpt" saver_pose.restore(sess, load_path) # load 3dmm shape and texture model from Tran et al.' CVPR2017 load_path = "./Shape_Model/ini_ShapeTextureNet_model.ckpt" saver_ini_shape_net.restore(sess, load_path) # load our expression net model load_path = "./Expression_Model/ini_exprNet_model.ckpt" saver_ini_expr_net.restore(sess, load_path) ## Modifed Basel Face Model BFM_path = './Shape_Model/BaselFaceModel_mod.mat' model = scipy.io.loadmat(BFM_path,squeeze_me=True,struct_as_record=False) model = model["BFM"] faces = model.faces-1 print('> Loaded the Basel Face Model to write the 3D output!') print('> Start to estimate Expression, Shape, and Pose!') with open(output_proc, 'r') as csvfile: csvreader = csv.reader(csvfile, delimiter=',') for row in csvreader: image_key = row[0] image_file_path = row[1] start = time.time() print('> Process ' + image_file_path) image = cv2.imread(image_file_path,1) # BGR image = np.asarray(image) # Fix the grey image if len(image.shape) < 3: image_r = np.reshape(image, (image.shape[0], image.shape[1], 1)) image = np.append(image_r, image_r, axis=2) image = np.append(image, image_r, axis=2) image = np.reshape(image, [1, FLAGS.image_size, FLAGS.image_size, 3]) (Shape_Texture, Expr, Pose) = sess.run([fc1ls, fc1le, pose_model.preds_unNormalized], feed_dict={x: image}) outFile = mesh_folder + '/' + image_key Pose = np.reshape(Pose, [-1]) Shape_Texture = np.reshape(Shape_Texture, [-1]) Shape = Shape_Texture[0:99] Shape = np.reshape(Shape, [-1]) Expr = np.reshape(Expr, [-1]) ######################################### ### Save 3D shape information (.ply file) ######################################### # Shape Only #S,T = utils.projectBackBFM(model,Shape_Texture) #utils.write_ply_textureless(outFile + '_ShapeOnly.ply', S, faces) # Shape + Expression #SE,TE = utils.projectBackBFM_withExpr(model, Shape_Texture, Expr) #utils.write_ply_textureless(outFile + '_Shape_and_Expr.ply', SE, faces) # Shape + Expression + Pose SEP,TEP = utils.projectBackBFM_withEP(model, Shape_Texture, Expr, Pose) utils.write_ply_textureless(outFile + '_Shape_Expr_Pose.ply', SEP, faces) end = time.time() print(end - start)
def extract_PSE_feats(): # Prepare data data_dict = myparse.parse_input('./input.csv') # please see input.csv for the input format print len(data_dict) ## Pre-processing the images print '> preproc' pu.preProcessImage(_tmpdir, data_dict, './', factor, _alexNetSize, output_proc) # placeholders for the batches x = tf.placeholder(tf.float32, [FLAGS.batch_size, FLAGS.image_size, FLAGS.image_size, 3]) ################### # Face Pose-Net ################### net_data = np.load("./fpn_new_model/PAM_frontal_ALexNet.npy").item() pose_labels = np.zeros([FLAGS.batch_size,6]) x1 = tf.image.resize_bilinear(x, tf.constant([227,227], dtype=tf.int32)) # Image normalization x1 = x1 / 255. # from [0,255] to [0,1] # subtract training mean mean = tf.reshape(train_mean_vec, [1, 1, 1, 3]) mean = tf.cast(mean, 'float32') x1 = x1 - mean pose_model = Pose_model.Pose_Estimation(x1, pose_labels, 'valid', 0, 1, 1, 0.01, net_data, FLAGS.batch_size, mean_labels, std_labels) pose_model._build_graph() del net_data ################### # Shape CNN ################### x2 = tf.image.resize_bilinear(x, tf.constant([224,224], dtype=tf.int32)) x2 = tf.cast(x2, 'float32') x2 = tf.reshape(x2, [FLAGS.batch_size, 224, 224, 3]) # Image normalization mean = tf.reshape(mean_image_shape, [1, 224, 224, 3]) mean = tf.cast(mean, 'float32') x2 = x2 - mean with tf.variable_scope('shapeCNN'): net_shape = resnet101_shape({'input': x2}, trainable=True) pool5 = net_shape.layers['pool5'] pool5 = tf.squeeze(pool5) pool5 = tf.reshape(pool5, [FLAGS.batch_size,-1]) npzfile = np.load('./ResNet/ShapeNet_fc_weights.npz') ini_weights_shape = npzfile['ini_weights_shape'] ini_biases_shape = npzfile['ini_biases_shape'] with tf.variable_scope('shapeCNN_fc1'): fc1ws = tf.Variable(tf.reshape(ini_weights_shape, [2048,-1]), trainable=True, name='weights') fc1bs = tf.Variable(tf.reshape(ini_biases_shape, [-1]), trainable=True, name='biases') fc1ls = tf.nn.bias_add(tf.matmul(pool5, fc1ws), fc1bs) ################### # Expression CNN ################### with tf.variable_scope('exprCNN'): net_expr = resnet101_expr({'input': x2}, trainable=True) pool5 = net_expr.layers['pool5'] pool5 = tf.squeeze(pool5) pool5 = tf.reshape(pool5, [FLAGS.batch_size,-1]) npzfile = np.load('./ResNet/ExpNet_fc_weights.npz') ini_weights_expr = npzfile['ini_weights_expr'] ini_biases_expr = npzfile['ini_biases_expr'] with tf.variable_scope('exprCNN_fc1'): fc1we = tf.Variable(tf.reshape(ini_weights_expr, [2048,29]), trainable=True, name='weights') fc1be = tf.Variable(tf.reshape(ini_biases_expr, [29]), trainable=True, name='biases') fc1le = tf.nn.bias_add(tf.matmul(pool5, fc1we), fc1be) # Add ops to save and restore all the variables. init_op = tf.global_variables_initializer() saver_pose = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Spatial_Transformer')) saver_ini_shape_net = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='shapeCNN')) saver_ini_expr_net = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='exprCNN')) with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: sess.run(init_op) # Load face pose net model from Chang et al.'ICCVW17 load_path = "./fpn_new_model/model_0_1.0_1.0_1e-07_1_16000.ckpt" saver_pose.restore(sess, load_path) # load 3dmm shape and texture model from Tran et al.' CVPR2017 load_path = "./Shape_Model/ini_ShapeTextureNet_model.ckpt" saver_ini_shape_net.restore(sess, load_path) # load our expression net model load_path = "./Expression_Model/ini_exprNet_model.ckpt" saver_ini_expr_net.restore(sess, load_path) ## Modifed Basel Face Model BFM_path = './Shape_Model/BaselFaceModel_mod.mat' model = scipy.io.loadmat(BFM_path,squeeze_me=True,struct_as_record=False) model = model["BFM"] faces = model.faces-1 print '> Loaded the Basel Face Model to write the 3D output!' print '> Start to estimate Expression, Shape, and Pose!' with open(output_proc, 'rb') as csvfile: csvreader = csv.reader(csvfile, delimiter=',') for row in csvreader: image_key = row[0] image_file_path = row[1] print '> Process ' + image_file_path image = cv2.imread(image_file_path,1) # BGR image = np.asarray(image) # Fix the grey image if len(image.shape) < 3: image_r = np.reshape(image, (image.shape[0], image.shape[1], 1)) image = np.append(image_r, image_r, axis=2) image = np.append(image, image_r, axis=2) image = np.reshape(image, [1, FLAGS.image_size, FLAGS.image_size, 3]) (Shape_Texture, Expr, Pose) = sess.run([fc1ls, fc1le, pose_model.preds_unNormalized], feed_dict={x: image}) outFile = mesh_folder + '/' + image_key Pose = np.reshape(Pose, [-1]) Shape_Texture = np.reshape(Shape_Texture, [-1]) Shape = Shape_Texture[0:99] Shape = np.reshape(Shape, [-1]) Expr = np.reshape(Expr, [-1]) ######################################### ### Save 3D shape information (.ply file) ######################################### # Shape Only #S,T = utils.projectBackBFM(model,Shape_Texture) #utils.write_ply_textureless(outFile + '_ShapeOnly.ply', S, faces) # Shape + Expression #SE,TE = utils.projectBackBFM_withExpr(model, Shape_Texture, Expr) #utils.write_ply_textureless(outFile + '_Shape_and_Expr.ply', SE, faces) # Shape + Expression + Pose SEP,TEP = utils.projectBackBFM_withEP(model, Shape_Texture, Expr, Pose) utils.write_ply_textureless(outFile + '_Shape_Expr_Pose.ply', SEP, faces)