def save_data(self, path, prefix):
attributes = [
self.pose, self.expression, self.illumination, self.make_up,
self.occlusion, self.blur
]
attributes = np.asarray(attributes, dtype=np.int32)
attributes_str = ' '.join(list(map(str, attributes)))
labels = []
TRACKED_POINTS = [
33, 38, 50, 46, 60, 64, 68, 72, 55, 59, 76, 82, 85, 16
]
for i, (img, lanmark) in enumerate(zip(self.imgs, self.landmarks)):
assert lanmark.shape == (98, 2)
save_path = os.path.join(path, prefix + '_' + str(i) + '.png')
assert not os.path.exists(save_path), save_path
cv2.imwrite(save_path, img)
euler_angles_landmark = []
for index in TRACKED_POINTS:
euler_angles_landmark.append(lanmark[index])
euler_angles_landmark = np.asarray(euler_angles_landmark).reshape(
(-1, 28))
pitch, yaw, roll = calculate_pitch_yaw_roll(
euler_angles_landmark[0])
euler_angles = np.asarray((pitch, yaw, roll), dtype=np.float32)
euler_angles_str = ' '.join(list(map(str, euler_angles)))
landmark_str = ' '.join(
list(map(str,
lanmark.reshape(-1).tolist())))
label = '{} {} {} {}\n'.format(save_path, landmark_str,
attributes_str, euler_angles_str)
labels.append(label)
return labels
def get_euler_angle_weights(landmarks_batch, euler_angles_pre, device):
TRACKED_POINTS = [17, 21, 22, 26, 36, 39, 42, 45, 31, 35, 48, 54, 57, 8]
euler_angles_landmarks = []
landmarks_batch = landmarks_batch.numpy()
for index in TRACKED_POINTS:
euler_angles_landmarks.append(landmarks_batch[:, 2 * index:2 * index + 2])
euler_angles_landmarks = np.asarray(euler_angles_landmarks).transpose((1, 0, 2)).reshape((-1, 28))
euler_angles_gt = []
for j in range(euler_angles_landmarks.shape[0]):
pitch, yaw, roll = calculate_pitch_yaw_roll(euler_angles_landmarks[j])
euler_angles_gt.append((pitch, yaw, roll))
euler_angles_gt = np.asarray(euler_angles_gt).reshape((-1, 3))
euler_angles_gt = torch.Tensor(euler_angles_gt).to(device)
euler_angle_weights = 1 - torch.cos(torch.abs(euler_angles_gt - euler_angles_pre))
euler_angle_weights = torch.sum(euler_angle_weights, 1)
return euler_angle_weights
def save_data(self, path, prefix):
# attributeは特にいじらず保存
attributes = [
self.pose, self.expression, self.illumination, self.make_up,
self.occlusion, self.blur
]
attributes = np.asarray(attributes, dtype=np.int32)
attributes_str = ' '.join(list(map(str, attributes)))
labels = []
for i, (img,
landmark_lip) in enumerate(zip(self.imgs, self.landmark_lips)):
save_path = os.path.join(path, prefix + '_' + str(i) + '.png')
assert not os.path.exists(save_path), save_path
# imgsにcrop画像を保存
cv2.imwrite(save_path, img)
# tracked pointsからpitch yaw rollを計算し保存
landmark = self.landmarks[i]
euler_angles_landmark = []
for index in self.tracked_points:
euler_angles_landmark.append([
landmark[index][0] * img.shape[0],
landmark[index][1] * img.shape[1]
])
euler_angles_landmark = np.asarray(euler_angles_landmark).reshape(
(-1, 28))
pitch, yaw, roll = calculate_pitch_yaw_roll(
euler_angles_landmark[0], self.image_size, self.image_size)
euler_angles = np.asarray((pitch, yaw, roll), dtype=np.float32)
euler_angles_str = ' '.join(list(map(str, euler_angles)))
landmark_lip_str = ' '.join(
list(map(str,
landmark_lip.reshape(-1).tolist())))
label = '{} {} {} {}\n'.format(save_path, landmark_lip_str,
attributes_str, euler_angles_str)
labels.append(label)
return labels
def train(sess, epoch_size, epoch, max_epoch, list_ops, merged,
summary_writer):
image_batch, landmarks_batch, attribute_batch = list_ops[
'train_next_element']
TRACKED_POINTS = [17, 21, 22, 26, 36, 39, 42, 45, 31, 35, 48, 54, 57, 8]
start_time = time.time()
for i in range(epoch_size):
# # print(images.shape, landmarks.shape, attributes.shape)
#TODO : get the w_n and euler_angles_gt_batch
images, landmarks, attributes = sess.run(
[image_batch, landmarks_batch, attribute_batch])
euler_angles_landmarks = []
for index in TRACKED_POINTS:
euler_angles_landmarks.append(landmarks[:,
2 * index:2 * index + 2])
euler_angles_landmarks = np.asarray(euler_angles_landmarks).transpose(
(1, 0, 2)).reshape((-1, 28))
euler_angles_gt = []
for j in range(euler_angles_landmarks.shape[0]):
pitch, yaw, roll = calculate_pitch_yaw_roll(
euler_angles_landmarks[j])
euler_angles_gt.append((pitch, yaw, roll))
euler_angles_gt = np.asarray(euler_angles_gt).reshape((-1, 3))
'''
calculate the w_n: return the batch [-1,1]
c :
#201: 表情(expression) 0->正常表情(normal expression) 1->夸张的表情(exaggerate expression)
#202: 照度(illumination) 0->正常照明(normal illumination) 1->极端照明(extreme illumination)
#203: 化妆(make-up) 0->无化妆(no make-up) 1->化妆(make-up)
#204: 遮挡(occlusion) 0->无遮挡(no occlusion) 1->遮挡(occlusion)
#205: 模糊(blur) 0->清晰(clear) 1->模糊(blur)
'''
feed_dict = {
list_ops['image_batch']: images,
list_ops['landmark_batch']: landmarks,
list_ops['attribute_batch']: attributes,
list_ops['phase_train_placeholder']: True,
list_ops['euler_angles_gt_batch']: euler_angles_gt
}
loss, _, lr, regularization_loss = sess.run([
list_ops['loss'], list_ops['train_op'], list_ops['lr_op'],
list_ops['regularization_loss']
],
feed_dict=feed_dict)
result_summary = sess.run(merged, feed_dict=feed_dict)
summary_writer.add_summary(result_summary, i + epoch_size * epoch)
if ((i + 1) % 100) == 0 or (i + 1) == epoch_size:
Epoch = 'Epoch:[{:<4}][{:<4}/{:<4}]'.format(
epoch, i + 1, epoch_size)
Loss = 'Loss: {:2.3f}\trg_loss:{:2.3f}'.format(
loss, regularization_loss)
average_time = (time.time() - start_time) / (i + 1)
remain_time = average_time * (epoch_size * max_epoch -
(i + epoch_size * epoch)) / 3600
print(
'{}\t{}\t lr {:2.3}\t average_time:{:.3f}s\t remain_time:{:.3f}h'
.format(Epoch, Loss, lr, average_time, remain_time))
def train(sess, epoch_size, epoch, list_ops):
image_batch, landmarks_batch, attribute_batch = list_ops[
'train_next_element']
TRACKED_POINTS = [33, 38, 50, 46, 60, 64, 68, 72, 55, 59, 76, 82, 85, 16]
for i in range(epoch_size):
# # print(images.shape, landmarks.shape, attributes.shape)
#TODO : get the w_n and euler_angles_gt_batch
images, landmarks, attributes = sess.run(
[image_batch, landmarks_batch, attribute_batch])
euler_angles_landmarks = []
for index in TRACKED_POINTS:
euler_angles_landmarks.append(landmarks[:,
2 * index:2 * index + 2])
euler_angles_landmarks = np.asarray(euler_angles_landmarks).reshape(
(-1, 28))
euler_angles_gt = []
for j in range(euler_angles_landmarks.shape[0]):
pitch, yaw, roll = calculate_pitch_yaw_roll(
euler_angles_landmarks[j])
euler_angles_gt.append((pitch, yaw, roll))
euler_angles_gt = np.asarray(euler_angles_gt).reshape((-1, 3))
'''
calculate the w_n: return the batch [-1,1]
c :
#201: 表情(expression) 0->正常表情(normal expression) 1->夸张的表情(exaggerate expression)
#202: 照度(illumination) 0->正常照明(normal illumination) 1->极端照明(extreme illumination)
#203: 化妆(make-up) 0->无化妆(no make-up) 1->化妆(make-up)
#204: 遮挡(occlusion) 0->无遮挡(no occlusion) 1->遮挡(occlusion)
#205: 模糊(blur) 0->清晰(clear) 1->模糊(blur)
'''
attributes_w_n = tf.to_float(attributes[:, 1:6])
_num = attributes_w_n.shape[0]
mat_ratio = tf.reduce_mean(attributes_w_n, axis=0)
#TODO when use function tf.map_fn get error results [inf,nan]
# mat_ratio = tf.map_fn(lambda x:1.0/x if not x==0.0 else 0.0,mat_ratio)
mat_ratio = map(
lambda x: 1.0 / x if not x == 0.0 else float(images.shape[0]),
sess.run(mat_ratio))
attributes_w_n = attributes_w_n * mat_ratio
attributes_w_n = tf.reduce_sum(attributes_w_n, axis=1)
# attributes_w_n = tf.expand_dims(attributes_w_n,1)
# attributes_w_n = tf.tile(attributes_w_n,[_num])
#TODO change the value of the zero in mat
attributes_w_n = sess.run(attributes_w_n)
feed_dict = {
list_ops['image_batch']: images,
list_ops['landmark_batch']: landmarks,
list_ops['attribute_batch']: attributes,
list_ops['phase_train_placeholder']: True,
list_ops['euler_angles_gt_batch']: euler_angles_gt,
list_ops['w_n']: attributes_w_n
}
loss, _, lr, L2_loss = sess.run([list_ops['loss'], list_ops['train_op'], list_ops['lr_op'],\
list_ops['L2_loss']], feed_dict=feed_dict)
if ((i + 1) % 100) == 0 or (i + 1) == epoch_size:
Epoch = 'Epoch:[{:<4}][{:<4}/{:<4}]'.format(
epoch, i + 1, epoch_size)
Loss = 'Loss {:2.3f}\tL2_loss {:2.3f}'.format(loss, L2_loss)
print('{}\t{}\t lr {:2.3}'.format(Epoch, Loss, lr))
def cal_hard_rotate_num(self, img, lands, repeat):
# 唇augは精度向上しない
# 縦横比で比較
lipmf_alpha = 30
# 唇両端: 48 - 54
d_wide = np.linalg.norm(lands[54] - lands[48])
# 唇上下: 51 - 57
d_vert = np.linalg.norm(lands[57] - lands[51])
aspect_rat = d_wide / d_vert
img_tmp = img.copy()
cv2.imwrite("./sample_lipaspect.jpg", img_tmp)
if (aspect_rat == 0) or (self.aspect_rat_mean_shape == 0):
aspect_expand = 100
elif aspect_rat >= self.aspect_rat_mean_shape:
aspect_expand = aspect_rat / self.aspect_rat_mean_shape
elif aspect_rat < self.aspect_rat_mean_shape:
aspect_expand = self.aspect_rat_mean_shape / aspect_rat
#print("aspect_expand: ", aspect_expand)
aspect_expand = (aspect_expand)**2
#print("aspect_expand: ", aspect_expand)
# 上唇の大きさ: 51 - 62
d_ulip = np.linalg.norm(lands[62] - lands[51])
if (d_ulip == 0) or (self.d_ulip_mean_face == 0):
ulip_expand = 100
elif d_ulip >= self.d_ulip_mean_face:
ulip_expand = d_ulip / self.d_ulip_mean_face
elif d_ulip < self.d_ulip_mean_face:
ulip_expand = self.d_ulip_mean_face / d_ulip
#print("ulip_expand: ", ulip_expand)
ulip_expand = (ulip_expand)**2
#print("ulip_expand: ", ulip_expand)
# 下唇の大きさ: 57 - 66
d_blip = np.linalg.norm(lands[57] - lands[66])
if (d_blip == 0) or (self.d_blip_mean_face == 0):
blip_expand = 100
elif d_blip >= self.d_blip_mean_face:
blip_expand = d_blip / self.d_blip_mean_face
elif d_blip < self.d_blip_mean_face:
blip_expand = self.d_blip_mean_face / d_blip
#print("blip_expand: ", blip_expand)
blip_expand = (blip_expand)**2
#print("blip_expand: ", blip_expand)
expand_rat = int(
(aspect_expand + ulip_expand + blip_expand) / lipmf_alpha)
if expand_rat > 100:
print("expand_rat: ", expand_rat)
expand_rat = 100
# sfed_repeat = repeat * expand_rat
global LIPMF_EXPAND
lipmf_expand = repeat * expand_rat
LIPMF_EXPAND += lipmf_expand
#print("sfed_repeat: ", sfed_repeat)
#import pdb; pdb.set_trace()
# tracked pointsからpitch yaw rollを計算し保存
euler_angles_landmark = []
for index in self.tracked_points:
euler_angles_landmark.append([
lands[index][0] * img.shape[0], lands[index][1] * img.shape[1]
])
euler_angles_landmark = np.asarray(euler_angles_landmark).reshape(
(-1, 28))
# (pitch = 上下, yaw = 横)
pitch, yaw, roll = calculate_pitch_yaw_roll(euler_angles_landmark[0],
self.image_size,
self.image_size)
# pitch(上下) is negative and smaller then num is bigger
if 5 <= pitch < 10:
pitch_sf = 4
elif 10 <= pitch < 20:
pitch_sf = 16
elif 20 <= pitch < 30:
pitch_sf = 48
elif 30 <= pitch < 40:
pitch_sf = 100
elif 40 <= pitch:
pitch_sf = 300
else:
pitch_sf = 1
#if pitch_sf > 29:
# print("pitch_sf: ", pitch_sf)
# img_tmp = img.copy()
# for x, y in (lands * self.image_size).astype(np.int32):
# cv2.circle(img_tmp, (x, y), 1, (255, 0, 0))
# cv2.imwrite("./sample_big_pitch_sf.jpg", img_tmp)
# #import pdb; pdb.set_trace()
# yaw(横) is bigger then num is bigger
if (-10 <= yaw < -5) or (5 <= yaw < 10):
yaw_sf = 2
elif (-20 <= yaw < -10) or (10 <= yaw < 20):
yaw_sf = 4
elif (-30 <= yaw < -20) or (20 <= yaw < 30):
yaw_sf = 8
elif (-40 <= yaw < -30) or (30 <= yaw < 40):
yaw_sf = 16
elif (yaw < -40) or (40 < yaw):
yaw_sf = 32
else:
yaw_sf = 1
#if yaw_sf > 15:
# print("yaw_sf: ", yaw_sf)
# img_tmp = img.copy()
# for x, y in (lands * self.image_size).astype(np.int32):
# cv2.circle(img_tmp, (x, y), 1, (255, 0, 0))
# cv2.imwrite("./sample_big_yaw_sf.jpg", img_tmp)
# #import pdb; pdb.set_trace()
# sfed_repeat = repeat * (pitch_sf + yaw_sf)
#print(pitch_sf)
#print(yaw_sf)
#print("sfed_repeat: ", sfed_repeat)
#if sfed_repeat > 300:
# print("big sf: ", sf)
# sfed_repeat = 300
# #print("big sfed_repeat: ", sfed_repeat)
# #img_tmp = img.copy()
# #for x, y in (lands * self.image_size).astype(np.int32):
# # cv2.circle(img_tmp, (x, y), 1, (255, 0, 0))
# #cv2.imwrite("./sample_big_totalsf.jpg", img_tmp)
global PITCHRAW_EXPAND
pitchraw_expand = repeat * (pitch_sf + yaw_sf)
PITCHRAW_EXPAND += pitchraw_expand
global EX_REPEAT_NUM
sfed_repeat = int((lipmf_expand * pitchraw_expand) / 2)
if sfed_repeat > 300:
print("big lipmf_expand: ", lipmf_expand)
print("big pitchraw_expand: ", pitchraw_expand)
print("big sfed_repeat: ", sfed_repeat)
sfed_repeat = 300
EX_REPEAT_NUM += sfed_repeat
return sfed_repeat
