def save_data(self, path, prefix): attributes = [ self.pose, self.expression, self.illumination, self.make_up, self.occlusion, self.blur ] attributes = np.asarray(attributes, dtype=np.int32) attributes_str = ' '.join(list(map(str, attributes))) labels = [] TRACKED_POINTS = [ 33, 38, 50, 46, 60, 64, 68, 72, 55, 59, 76, 82, 85, 16 ] for i, (img, lanmark) in enumerate(zip(self.imgs, self.landmarks)): assert lanmark.shape == (98, 2) save_path = os.path.join(path, prefix + '_' + str(i) + '.png') assert not os.path.exists(save_path), save_path cv2.imwrite(save_path, img) euler_angles_landmark = [] for index in TRACKED_POINTS: euler_angles_landmark.append(lanmark[index]) euler_angles_landmark = np.asarray(euler_angles_landmark).reshape( (-1, 28)) pitch, yaw, roll = calculate_pitch_yaw_roll( euler_angles_landmark[0]) euler_angles = np.asarray((pitch, yaw, roll), dtype=np.float32) euler_angles_str = ' '.join(list(map(str, euler_angles))) landmark_str = ' '.join( list(map(str, lanmark.reshape(-1).tolist()))) label = '{} {} {} {}\n'.format(save_path, landmark_str, attributes_str, euler_angles_str) labels.append(label) return labels
def get_euler_angle_weights(landmarks_batch, euler_angles_pre, device): TRACKED_POINTS = [17, 21, 22, 26, 36, 39, 42, 45, 31, 35, 48, 54, 57, 8] euler_angles_landmarks = [] landmarks_batch = landmarks_batch.numpy() for index in TRACKED_POINTS: euler_angles_landmarks.append(landmarks_batch[:, 2 * index:2 * index + 2]) euler_angles_landmarks = np.asarray(euler_angles_landmarks).transpose((1, 0, 2)).reshape((-1, 28)) euler_angles_gt = [] for j in range(euler_angles_landmarks.shape[0]): pitch, yaw, roll = calculate_pitch_yaw_roll(euler_angles_landmarks[j]) euler_angles_gt.append((pitch, yaw, roll)) euler_angles_gt = np.asarray(euler_angles_gt).reshape((-1, 3)) euler_angles_gt = torch.Tensor(euler_angles_gt).to(device) euler_angle_weights = 1 - torch.cos(torch.abs(euler_angles_gt - euler_angles_pre)) euler_angle_weights = torch.sum(euler_angle_weights, 1) return euler_angle_weights
def save_data(self, path, prefix): # attributeは特にいじらず保存 attributes = [ self.pose, self.expression, self.illumination, self.make_up, self.occlusion, self.blur ] attributes = np.asarray(attributes, dtype=np.int32) attributes_str = ' '.join(list(map(str, attributes))) labels = [] for i, (img, landmark_lip) in enumerate(zip(self.imgs, self.landmark_lips)): save_path = os.path.join(path, prefix + '_' + str(i) + '.png') assert not os.path.exists(save_path), save_path # imgsにcrop画像を保存 cv2.imwrite(save_path, img) # tracked pointsからpitch yaw rollを計算し保存 landmark = self.landmarks[i] euler_angles_landmark = [] for index in self.tracked_points: euler_angles_landmark.append([ landmark[index][0] * img.shape[0], landmark[index][1] * img.shape[1] ]) euler_angles_landmark = np.asarray(euler_angles_landmark).reshape( (-1, 28)) pitch, yaw, roll = calculate_pitch_yaw_roll( euler_angles_landmark[0], self.image_size, self.image_size) euler_angles = np.asarray((pitch, yaw, roll), dtype=np.float32) euler_angles_str = ' '.join(list(map(str, euler_angles))) landmark_lip_str = ' '.join( list(map(str, landmark_lip.reshape(-1).tolist()))) label = '{} {} {} {}\n'.format(save_path, landmark_lip_str, attributes_str, euler_angles_str) labels.append(label) return labels
def train(sess, epoch_size, epoch, max_epoch, list_ops, merged, summary_writer): image_batch, landmarks_batch, attribute_batch = list_ops[ 'train_next_element'] TRACKED_POINTS = [17, 21, 22, 26, 36, 39, 42, 45, 31, 35, 48, 54, 57, 8] start_time = time.time() for i in range(epoch_size): # # print(images.shape, landmarks.shape, attributes.shape) #TODO : get the w_n and euler_angles_gt_batch images, landmarks, attributes = sess.run( [image_batch, landmarks_batch, attribute_batch]) euler_angles_landmarks = [] for index in TRACKED_POINTS: euler_angles_landmarks.append(landmarks[:, 2 * index:2 * index + 2]) euler_angles_landmarks = np.asarray(euler_angles_landmarks).transpose( (1, 0, 2)).reshape((-1, 28)) euler_angles_gt = [] for j in range(euler_angles_landmarks.shape[0]): pitch, yaw, roll = calculate_pitch_yaw_roll( euler_angles_landmarks[j]) euler_angles_gt.append((pitch, yaw, roll)) euler_angles_gt = np.asarray(euler_angles_gt).reshape((-1, 3)) ''' calculate the w_n: return the batch [-1,1] c : #201: 表情(expression) 0->正常表情(normal expression) 1->夸张的表情(exaggerate expression) #202: 照度(illumination) 0->正常照明(normal illumination) 1->极端照明(extreme illumination) #203: 化妆(make-up) 0->无化妆(no make-up) 1->化妆(make-up) #204: 遮挡(occlusion) 0->无遮挡(no occlusion) 1->遮挡(occlusion) #205: 模糊(blur) 0->清晰(clear) 1->模糊(blur) ''' feed_dict = { list_ops['image_batch']: images, list_ops['landmark_batch']: landmarks, list_ops['attribute_batch']: attributes, list_ops['phase_train_placeholder']: True, list_ops['euler_angles_gt_batch']: euler_angles_gt } loss, _, lr, regularization_loss = sess.run([ list_ops['loss'], list_ops['train_op'], list_ops['lr_op'], list_ops['regularization_loss'] ], feed_dict=feed_dict) result_summary = sess.run(merged, feed_dict=feed_dict) summary_writer.add_summary(result_summary, i + epoch_size * epoch) if ((i + 1) % 100) == 0 or (i + 1) == epoch_size: Epoch = 'Epoch:[{:<4}][{:<4}/{:<4}]'.format( epoch, i + 1, epoch_size) Loss = 'Loss: {:2.3f}\trg_loss:{:2.3f}'.format( loss, regularization_loss) average_time = (time.time() - start_time) / (i + 1) remain_time = average_time * (epoch_size * max_epoch - (i + epoch_size * epoch)) / 3600 print( '{}\t{}\t lr {:2.3}\t average_time:{:.3f}s\t remain_time:{:.3f}h' .format(Epoch, Loss, lr, average_time, remain_time))
def train(sess, epoch_size, epoch, list_ops): image_batch, landmarks_batch, attribute_batch = list_ops[ 'train_next_element'] TRACKED_POINTS = [33, 38, 50, 46, 60, 64, 68, 72, 55, 59, 76, 82, 85, 16] for i in range(epoch_size): # # print(images.shape, landmarks.shape, attributes.shape) #TODO : get the w_n and euler_angles_gt_batch images, landmarks, attributes = sess.run( [image_batch, landmarks_batch, attribute_batch]) euler_angles_landmarks = [] for index in TRACKED_POINTS: euler_angles_landmarks.append(landmarks[:, 2 * index:2 * index + 2]) euler_angles_landmarks = np.asarray(euler_angles_landmarks).reshape( (-1, 28)) euler_angles_gt = [] for j in range(euler_angles_landmarks.shape[0]): pitch, yaw, roll = calculate_pitch_yaw_roll( euler_angles_landmarks[j]) euler_angles_gt.append((pitch, yaw, roll)) euler_angles_gt = np.asarray(euler_angles_gt).reshape((-1, 3)) ''' calculate the w_n: return the batch [-1,1] c : #201: 表情(expression) 0->正常表情(normal expression) 1->夸张的表情(exaggerate expression) #202: 照度(illumination) 0->正常照明(normal illumination) 1->极端照明(extreme illumination) #203: 化妆(make-up) 0->无化妆(no make-up) 1->化妆(make-up) #204: 遮挡(occlusion) 0->无遮挡(no occlusion) 1->遮挡(occlusion) #205: 模糊(blur) 0->清晰(clear) 1->模糊(blur) ''' attributes_w_n = tf.to_float(attributes[:, 1:6]) _num = attributes_w_n.shape[0] mat_ratio = tf.reduce_mean(attributes_w_n, axis=0) #TODO when use function tf.map_fn get error results [inf,nan] # mat_ratio = tf.map_fn(lambda x:1.0/x if not x==0.0 else 0.0,mat_ratio) mat_ratio = map( lambda x: 1.0 / x if not x == 0.0 else float(images.shape[0]), sess.run(mat_ratio)) attributes_w_n = attributes_w_n * mat_ratio attributes_w_n = tf.reduce_sum(attributes_w_n, axis=1) # attributes_w_n = tf.expand_dims(attributes_w_n,1) # attributes_w_n = tf.tile(attributes_w_n,[_num]) #TODO change the value of the zero in mat attributes_w_n = sess.run(attributes_w_n) feed_dict = { list_ops['image_batch']: images, list_ops['landmark_batch']: landmarks, list_ops['attribute_batch']: attributes, list_ops['phase_train_placeholder']: True, list_ops['euler_angles_gt_batch']: euler_angles_gt, list_ops['w_n']: attributes_w_n } loss, _, lr, L2_loss = sess.run([list_ops['loss'], list_ops['train_op'], list_ops['lr_op'],\ list_ops['L2_loss']], feed_dict=feed_dict) if ((i + 1) % 100) == 0 or (i + 1) == epoch_size: Epoch = 'Epoch:[{:<4}][{:<4}/{:<4}]'.format( epoch, i + 1, epoch_size) Loss = 'Loss {:2.3f}\tL2_loss {:2.3f}'.format(loss, L2_loss) print('{}\t{}\t lr {:2.3}'.format(Epoch, Loss, lr))
def cal_hard_rotate_num(self, img, lands, repeat): # 唇augは精度向上しない # 縦横比で比較 lipmf_alpha = 30 # 唇両端: 48 - 54 d_wide = np.linalg.norm(lands[54] - lands[48]) # 唇上下: 51 - 57 d_vert = np.linalg.norm(lands[57] - lands[51]) aspect_rat = d_wide / d_vert img_tmp = img.copy() cv2.imwrite("./sample_lipaspect.jpg", img_tmp) if (aspect_rat == 0) or (self.aspect_rat_mean_shape == 0): aspect_expand = 100 elif aspect_rat >= self.aspect_rat_mean_shape: aspect_expand = aspect_rat / self.aspect_rat_mean_shape elif aspect_rat < self.aspect_rat_mean_shape: aspect_expand = self.aspect_rat_mean_shape / aspect_rat #print("aspect_expand: ", aspect_expand) aspect_expand = (aspect_expand)**2 #print("aspect_expand: ", aspect_expand) # 上唇の大きさ: 51 - 62 d_ulip = np.linalg.norm(lands[62] - lands[51]) if (d_ulip == 0) or (self.d_ulip_mean_face == 0): ulip_expand = 100 elif d_ulip >= self.d_ulip_mean_face: ulip_expand = d_ulip / self.d_ulip_mean_face elif d_ulip < self.d_ulip_mean_face: ulip_expand = self.d_ulip_mean_face / d_ulip #print("ulip_expand: ", ulip_expand) ulip_expand = (ulip_expand)**2 #print("ulip_expand: ", ulip_expand) # 下唇の大きさ: 57 - 66 d_blip = np.linalg.norm(lands[57] - lands[66]) if (d_blip == 0) or (self.d_blip_mean_face == 0): blip_expand = 100 elif d_blip >= self.d_blip_mean_face: blip_expand = d_blip / self.d_blip_mean_face elif d_blip < self.d_blip_mean_face: blip_expand = self.d_blip_mean_face / d_blip #print("blip_expand: ", blip_expand) blip_expand = (blip_expand)**2 #print("blip_expand: ", blip_expand) expand_rat = int( (aspect_expand + ulip_expand + blip_expand) / lipmf_alpha) if expand_rat > 100: print("expand_rat: ", expand_rat) expand_rat = 100 # sfed_repeat = repeat * expand_rat global LIPMF_EXPAND lipmf_expand = repeat * expand_rat LIPMF_EXPAND += lipmf_expand #print("sfed_repeat: ", sfed_repeat) #import pdb; pdb.set_trace() # tracked pointsからpitch yaw rollを計算し保存 euler_angles_landmark = [] for index in self.tracked_points: euler_angles_landmark.append([ lands[index][0] * img.shape[0], lands[index][1] * img.shape[1] ]) euler_angles_landmark = np.asarray(euler_angles_landmark).reshape( (-1, 28)) # (pitch = 上下, yaw = 横) pitch, yaw, roll = calculate_pitch_yaw_roll(euler_angles_landmark[0], self.image_size, self.image_size) # pitch(上下) is negative and smaller then num is bigger if 5 <= pitch < 10: pitch_sf = 4 elif 10 <= pitch < 20: pitch_sf = 16 elif 20 <= pitch < 30: pitch_sf = 48 elif 30 <= pitch < 40: pitch_sf = 100 elif 40 <= pitch: pitch_sf = 300 else: pitch_sf = 1 #if pitch_sf > 29: # print("pitch_sf: ", pitch_sf) # img_tmp = img.copy() # for x, y in (lands * self.image_size).astype(np.int32): # cv2.circle(img_tmp, (x, y), 1, (255, 0, 0)) # cv2.imwrite("./sample_big_pitch_sf.jpg", img_tmp) # #import pdb; pdb.set_trace() # yaw(横) is bigger then num is bigger if (-10 <= yaw < -5) or (5 <= yaw < 10): yaw_sf = 2 elif (-20 <= yaw < -10) or (10 <= yaw < 20): yaw_sf = 4 elif (-30 <= yaw < -20) or (20 <= yaw < 30): yaw_sf = 8 elif (-40 <= yaw < -30) or (30 <= yaw < 40): yaw_sf = 16 elif (yaw < -40) or (40 < yaw): yaw_sf = 32 else: yaw_sf = 1 #if yaw_sf > 15: # print("yaw_sf: ", yaw_sf) # img_tmp = img.copy() # for x, y in (lands * self.image_size).astype(np.int32): # cv2.circle(img_tmp, (x, y), 1, (255, 0, 0)) # cv2.imwrite("./sample_big_yaw_sf.jpg", img_tmp) # #import pdb; pdb.set_trace() # sfed_repeat = repeat * (pitch_sf + yaw_sf) #print(pitch_sf) #print(yaw_sf) #print("sfed_repeat: ", sfed_repeat) #if sfed_repeat > 300: # print("big sf: ", sf) # sfed_repeat = 300 # #print("big sfed_repeat: ", sfed_repeat) # #img_tmp = img.copy() # #for x, y in (lands * self.image_size).astype(np.int32): # # cv2.circle(img_tmp, (x, y), 1, (255, 0, 0)) # #cv2.imwrite("./sample_big_totalsf.jpg", img_tmp) global PITCHRAW_EXPAND pitchraw_expand = repeat * (pitch_sf + yaw_sf) PITCHRAW_EXPAND += pitchraw_expand global EX_REPEAT_NUM sfed_repeat = int((lipmf_expand * pitchraw_expand) / 2) if sfed_repeat > 300: print("big lipmf_expand: ", lipmf_expand) print("big pitchraw_expand: ", pitchraw_expand) print("big sfed_repeat: ", sfed_repeat) sfed_repeat = 300 EX_REPEAT_NUM += sfed_repeat return sfed_repeat