def print_histogram_plt_x_y(k, type, landmarks_arr):
import matplotlib.ticker as ticker
import random
image_utility = ImageUtility()
num_of_bins = 20
plt.figure()
for lndm in landmarks_arr:
landmark_arr_xy, landmark_arr_x, landmark_arr_y = image_utility.create_landmarks(landmarks=lndm,
scale_factor_x=1,
scale_factor_y=1)
data = landmark_arr_x
data = landmark_arr_y
# plt.plot(landmark_arr_x[0:32], landmark_arr_y[0:32], '-ok', c=color)
plt.hist(data, bins=num_of_bins, color='blue', edgecolor='blue', alpha=0.3)
# plt.hist2d(landmark_arr_x, landmark_arr_y, bins=num_of_bins, color='blue', edgecolor='black', alpha=0.5)
# plt.axis.set_major_formatter(ticker.PercentFormatter(xmax=len(landmark_arr_x)))
plt.savefig('histo_X_' + str(type) + '_' + str(k) + '.png', bbox_inches='tight')
plt.clf()
'''for Y'''
plt.figure()
for lndm in landmarks_arr:
landmark_arr_xy, landmark_arr_x, landmark_arr_y = image_utility.create_landmarks(landmarks=lndm,
scale_factor_x=1,
scale_factor_y=1)
data = landmark_arr_y
# plt.plot(landmark_arr_x[0:32], landmark_arr_y[0:32], '-ok', c=color)
plt.hist(data, bins=num_of_bins, color='green', edgecolor='green', alpha=0.3)
# plt.axis.set_major_formatter(ticker.PercentFormatter(xmax=len(data)))
plt.savefig('histo_Y_' + str(type) + '_' + str(k) + '.png', bbox_inches='tight')
plt.clf()
def print_partial(counter, img, landmarks_arr):
image_utility = ImageUtility()
# plt.figure()
# plt.imshow(img)
# implot = plt.imshow(img)
index =0
for lndm in landmarks_arr:
image = Image.new("L", (InputDataSize.image_input_size // 4, InputDataSize.image_input_size // 4))
draw = ImageDraw.Draw(image)
# color = "#"+''.join([random.choice('0123456789ABCDEF') for j in range(6)])
landmark_arr_xy, landmark_arr_x, landmark_arr_y = image_utility.create_landmarks_from_normalized(lndm, 224, 224, 112, 112)
# plt.scatter(x=landmark_arr_x[:], y=landmark_arr_y[:], c=color, s=20)
# plt.plot(landmark_arr_x, landmark_arr_y, '-ok', c=color)
# if index == 4 or index == 5 or index == 6 or index == 7:
# draw.polygon((landmark_arr_xy), fill='#ffffff')
landmark_arr_xy = (np.array(landmark_arr_xy) // 4).tolist()
draw.line((landmark_arr_xy), fill='#ffffff', width=2)
img_np = np.asarray(image)
plt.imshow(img_np)
image.save('0_name_' + str(counter) + '_' + str(index) + '.png')
index += 1
def test_result_per_image(k, img):
image_utility = ImageUtility()
pose_predicted = []
#image = np.expand_dims(img, axis=0)
#predict = model.forward(image)
heatmap_main = img
for i in range(14):
image_utility.print_image_arr_heat(k * 100 + i + 1, heatmap_main[i])
def test(self):
image_utility = ImageUtility()
cnn = CNNModel()
images_dir_out_ch = '/media/ali/extradata/facial_landmark_ds/from_ibug/test_set/challenging_intermediate_img/'
model = cnn.mobileNet_v2_small(tensor=None)
model.load_weights('weights-03-0.00095.h5')
counter = 0
for img_file in os.listdir(images_dir_out_ch):
if img_file.endswith("_img.npy"):
lbl_file = img_file[:-8] + "_lbl.npy"
main_img_file = img_file[:-8] + ".jpg"
img = Image.open(images_dir_out_ch + main_img_file)
image = np.expand_dims(load(images_dir_out_ch + img_file),
axis=0)
lbl = load(images_dir_out_ch + lbl_file)
p_lbl = model.predict(image)[0]
labels_predict_transformed, landmark_arr_x_p, landmark_arr_y_p = image_utility.\
create_landmarks_from_normalized(p_lbl, 224, 224, 112, 112)
imgpr.print_image_arr((counter + 1) * 1000, img,
landmark_arr_x_p, landmark_arr_y_p)
counter += 1
def __init__(self,
dataset_name,
arch,
num_output_layers,
weight_fname,
has_pose=False):
self.dataset_name = dataset_name
self.has_pose = has_pose
if dataset_name == DatasetName.w300:
self.output_len = D300wConf.num_of_landmarks * 2
elif dataset_name == DatasetName.cofw:
self.output_len = CofwConf.num_of_landmarks * 2
elif dataset_name == DatasetName.wflw:
self.output_len = WflwConf.num_of_landmarks * 2
cnn = CNNModel()
model = cnn.get_model(arch=arch,
input_tensor=None,
output_len=self.output_len)
model.load_weights(weight_fname)
img = None # load a cropped image
image_utility = ImageUtility()
pose_predicted = []
image = np.expand_dims(img, axis=0)
pose_predicted = model.predict(image)[1][0]
def __ASM(self, input_tensor, pca_postfix):
print(pca_postfix)
pca_utility = PCAUtility()
image_utility = ImageUtility()
eigenvalues, eigenvectors, meanvector = pca_utility.load_pca_obj(
DatasetName.ibug, pca_postfix=pca_postfix)
input_vector_batch = K.eval(input_tensor)
out_asm_vector = []
for i in range(LearningConfig.batch_size):
b_vector_p = self.calculate_b_vector(input_vector_batch[i], True,
eigenvalues, eigenvectors,
meanvector)
# asm_vector = meanvector + np.dot(eigenvectors, b_vector_p)
#
# labels_predict_transformed, landmark_arr_x_p, landmark_arr_y_p = \
# image_utility.create_landmarks_from_normalized(asm_vector, 224, 224, 112, 112)
# imgpr.print_image_arr(i + 1, np.zeros(shape=[224,224,3]), landmark_arr_x_p, landmark_arr_y_p)
out_asm_vector.append(meanvector +
np.dot(eigenvectors, b_vector_p))
out_asm_vector = np.array(out_asm_vector)
tensor_out = K.variable(out_asm_vector)
return tensor_out
def print_histogram_plt(k, type, landmarks_arr):
import matplotlib.ticker as ticker
import random
image_utility = ImageUtility()
# var = np.var(landmarks_arr, axis=0)
# mean = np.mean(landmarks_arr, axis=0)
#
plt.figure()
for lndm in landmarks_arr:
data = lndm
color = '#b52b65'
if type == 'face':
data = lndm[0:64]
color = '#ed6663'
# color = "#"+''.join([random.choice('0123456789ABCDEF') for j in range(6)])
# plt.plot(data, '-ok', c='#799351')
plt.hist(data, bins=20, color=color, alpha=0.3, histtype='bar')
# plt.hist(data, bins=num_of_bins, color=color, edgecolor='green', alpha=0.2)
# plt.axis.set_major_formatter(ticker.PercentFormatter(xmax=len(landmark_arr_x)))
# plt.text(0,15, 'mean: '+str(mean)+', var:' + str(var))
plt.savefig('histo_' + str(type) + '_' + str(k) + '.png', bbox_inches='tight')
plt.clf()
def _points_to_2d_face_graph(self, _points):
"""
:param points: [bs, 136]
:return: [bs, 56, 56, num_fg]
"""
'''rescale points'''
points = np.zeros([LearningConfig.batch_size, self.num_landmark])
image_utility = ImageUtility()
indx = 0
for item in _points:
point_scaled, px_1, py_1 = image_utility.create_landmarks_from_normalized(
item, InputDataSize.hm_size, InputDataSize.hm_size,
InputDataSize.hm_center, InputDataSize.hm_center)
# imgpr.print_image_arr('pts_' + str(indx), np.zeros([56,56]), px_1, py_1)
points[indx, :] = point_scaled
indx += 1
'''create partial parts: '''
partial_points = self._slice_face_graph_np(points)
'''convert from flatten to 2d'''
points_2d = []
for pnt in partial_points:
points_2d.append(
pnt.reshape([LearningConfig.batch_size,
len(pnt[1]) // 2, 2]))
'''create the spare img for each facial part:'''
hm_img = np.zeros([
LearningConfig.batch_size, InputDataSize.hm_size,
InputDataSize.hm_size, self.num_face_graph_elements
])
# bs, 12 * 2
for i in range(LearningConfig.batch_size):
for j in range(self.num_face_graph_elements):
t_hm = np.zeros([InputDataSize.hm_size, InputDataSize.hm_size])
for x_y in points_2d[j][i]:
if not (0 <= x_y[0] <= InputDataSize.hm_size - 1):
x_y[0] = 0
if not (0 <= x_y[1] <= InputDataSize.hm_size - 1):
x_y[1] = 0
t_hm[int(x_y[1]), int(x_y[0])] = 1
hm_img[i, :, :, j] = t_hm
return hm_img
def _points_to_2d(self, _points):
"""
:param _points:
:return:
"""
tf_rec = TFRecordUtility(self.num_landmark)
image_utility = ImageUtility()
hm_arr = []
for i in range(LearningConfig.batch_size):
_x_y, _x, _y = image_utility.create_landmarks_from_normalized(
_points[i], InputDataSize.image_input_size,
InputDataSize.image_input_size, InputDataSize.img_center,
InputDataSize.img_center)
hm_multi_layer = tf_rec.generate_hm(InputDataSize.hm_size,
InputDataSize.hm_size,
np.array(_x_y),
self.hm_stride / 2, False)
hm = np.sum(hm_multi_layer, axis=2)
hm_arr.append(hm)
return np.array(hm_arr)
def test_pca_validity(self, pca_postfix):
cnn_model = CNNModel()
pca_utility = PCAUtility()
tf_record_utility = TFRecordUtility()
image_utility = ImageUtility()
eigenvalues, eigenvectors, meanvector = pca_utility.load_pca_obj(
dataset_name=DatasetName.ibug, pca_postfix=pca_postfix)
lbl_arr, img_arr, pose_arr = tf_record_utility.retrieve_tf_record(
tfrecord_filename=IbugConf.tf_train_path,
number_of_records=30,
only_label=False)
for i in range(20):
b_vector_p = self.calculate_b_vector(lbl_arr[i], True, eigenvalues,
eigenvectors, meanvector)
lbl_new = meanvector + np.dot(eigenvectors, b_vector_p)
labels_true_transformed, landmark_arr_x_t, landmark_arr_y_t = image_utility. \
create_landmarks_from_normalized(lbl_arr[i], 224, 224, 112, 112)
labels_true_transformed_pca, landmark_arr_x_pca, landmark_arr_y_pca = image_utility. \
create_landmarks_from_normalized(lbl_new, 224, 224, 112, 112)
image_utility.print_image_arr(i, img_arr[i], landmark_arr_x_t,
landmark_arr_y_t)
image_utility.print_image_arr(i * 1000, img_arr[i],
landmark_arr_x_pca,
landmark_arr_y_pca)
def test_pca_validity(self, dataset_name, pca_postfix):
image_utility = ImageUtility()
eigenvalues = load('pca_obj/' + dataset_name + self.eigenvalues_prefix + str(pca_postfix)+".npy")
eigenvectors = load('pca_obj/' + dataset_name + self.eigenvectors_prefix + str(pca_postfix)+".npy")
meanvector = load('pca_obj/' + dataset_name + self.meanvector_prefix + str(pca_postfix)+".npy")
'''load data: '''
lbl_arr = []
img_arr = []
path = None
if dataset_name == DatasetName.ibug:
path = IbugConf.rotated_img_path_prefix # rotated is ok, since advs_aug is the same as rotated
num_of_landmarks = IbugConf.num_of_landmarks
elif dataset_name == DatasetName.cofw:
path = CofwConf.rotated_img_path_prefix
num_of_landmarks = CofwConf.num_of_landmarks
elif dataset_name == DatasetName.wflw:
path = WflwConf.rotated_img_path_prefix
num_of_landmarks = WflwConf.num_of_landmarks
for file in tqdm(os.listdir(path)):
if file.endswith(".pts"):
pts_file = os.path.join(path, file)
img_file = pts_file[:-3] + "jpg"
if not os.path.exists(img_file):
continue
points_arr = []
with open(pts_file) as fp:
line = fp.readline()
cnt = 1
while line:
if 3 < cnt <= num_of_landmarks+3:
x_y_pnt = line.strip()
x = float(x_y_pnt.split(" ")[0])
y = float(x_y_pnt.split(" ")[1])
points_arr.append(x)
points_arr.append(y)
line = fp.readline()
cnt += 1
lbl_arr.append(points_arr)
img_arr.append(Image.open(img_file))
for i in range(30):
b_vector_p = self.calculate_b_vector(lbl_arr[i], True, eigenvalues, eigenvectors, meanvector)
lbl_new = meanvector + np.dot(eigenvectors, b_vector_p)
lbl_new = lbl_new.tolist()
labels_true_transformed, landmark_arr_x_t, landmark_arr_y_t = image_utility. \
create_landmarks(lbl_arr[i], 1, 1)
labels_true_transformed_pca, landmark_arr_x_pca, landmark_arr_y_pca = image_utility. \
create_landmarks(lbl_new, 1, 1)
image_utility.print_image_arr(i, img_arr[i], landmark_arr_x_t, landmark_arr_y_t)
image_utility.print_image_arr(i * 1000, img_arr[i], landmark_arr_x_pca, landmark_arr_y_pca)
def print_arr(k, type, landmarks_arr):
import random
image_utility = ImageUtility()
plt.figure()
for lndm in landmarks_arr:
color = "#"+''.join([random.choice('0123456789ABCDEF') for j in range(6)])
landmark_arr_xy, landmark_arr_x, landmark_arr_y = image_utility.create_landmarks(landmarks=lndm,
scale_factor_x=1,
scale_factor_y=1)
if type == 'full':
plt.scatter(x=landmark_arr_x[:], y=landmark_arr_y[:], c=color, s=2)
elif type == 'face':
plt.scatter(x=landmark_arr_x[0:32], y=landmark_arr_y[0:32], c=color, s=5)
plt.plot(landmark_arr_x[0:32], landmark_arr_y[0:32], '-ok', c=color)
elif type == 'eyes':
plt.scatter(x=landmark_arr_x[60:75], y=landmark_arr_y[60:75], c=color, s=5)
plt.plot(landmark_arr_x[60:75], landmark_arr_y[60:75], '-ok', c=color)
elif type == 'nose':
plt.scatter(x=landmark_arr_x[51:59], y=landmark_arr_y[51:59], c=color, s=5)
plt.plot(landmark_arr_x[51:59], landmark_arr_y[51:59], '-ok', c=color)
elif type == 'mouth':
plt.scatter(x=landmark_arr_x[76:95], y=landmark_arr_y[76:95], c=color, s=5)
plt.plot(landmark_arr_x[76:95], landmark_arr_y[76:95], '-ok', c=color)
elif type == 'eyebrow':
plt.scatter(x=landmark_arr_x[33:50], y=landmark_arr_y[33:50], c=color, s=5)
plt.plot(landmark_arr_x[33:50], landmark_arr_y[33:50], '-ok', c=color)
plt.axis('off')
plt.savefig('name_' + str(type) + '_' + str(k) + '.png', bbox_inches='tight')
# plt.show()
plt.clf()
def print_hm_cord(self, epoch, step, images, heatmaps_gr, heatmaps_pr,
points_gr, points_pr):
epoch = 0
image_utility = ImageUtility()
for i in range(LearningConfig.batch_size):
# imgpr.print_image_arr_heat('hgr' + '.' + str(epoch)+"."+str(step)+'.'+str(i), heatmaps_gr[i])
imgpr.print_image_arr_heat(
'hpr' + '.' + str(epoch) + "." + str(step) + '.' + str(i),
heatmaps_pr[i])
# landmark_arr_flat_n, landmark_arr_x_n, landmark_arr_y_n = image_utility.create_landmarks_from_normalized(
# points_gr[i], 224, 224, 112, 112)
# imgpr.print_image_arr('cgr' + '.' + str(epoch)+"."+str(step)+'.'+str(i), np.zeros([224, 224,3]), landmark_arr_x_n,
# landmark_arr_y_n)
landmark_arr_flat_n, landmark_arr_x_n, landmark_arr_y_n = image_utility.create_landmarks_from_normalized(
points_pr[i], InputDataSize.image_input_size,
InputDataSize.image_input_size,
InputDataSize.image_input_size // 2,
InputDataSize.image_input_size // 2)
imgpr.print_image_arr(
'cpr' + '.' + str(epoch) + "." + str(step) + '.' + str(i),
np.zeros([
InputDataSize.image_input_size,
InputDataSize.image_input_size, 3
]), landmark_arr_x_n, landmark_arr_y_n)
pts_path_prefix = '/media/data/Soroush_data/body_tracking/FLIC/FLIC/imagesAnnotaion_train/'
# pts_path_prefix = '/home/soroush/PycharmProjects/Bodytracking/dataloader/FLIC/imagesAnnotaion_train/'
# pts_path_prefix = '/home/soroush/PycharmProjects/Bodytracking/dataloader/LSP/lsp_dataset_original/points/'
rotated_img_path_prefix = '/media/data/Soroush_data/body_tracking/FLIC/FLIC/images_rotated/'
# rotated_img_path_prefix = '/home/soroush/PycharmProjects/Bodytracking/dataloader/FLIC/images_rotated/'
rotated_pts_path_prefix = '/media/data/Soroush_data/body_tracking/FLIC/FLIC/points_rotated/'
# rotated_pts_path_prefix = '/home/soroush/PycharmProjects/Bodytracking/dataloader/FLIC/points_rotated/'
# before_heatmap_img_path_prefix = '/home/soroush/PycharmProjects/Bodytracking/dataloader/LSP/lsp_dataset_original/heatmap/'
# before_heatmap_img_path_prefix = '/home/soroush/PycharmProjects/Bodytracking/dataloader/FLIC/heatmap/'
before_heatmap_img_path_prefix = '/media/data/Soroush_data/body_tracking/FLIC/FLIC/heatmap/'
image_utility = ImageUtility()
"""
import random
png_file_arr = []
png_file_name = []
for file in sorted(os.listdir(IbugConf.pts_path_prefix)):
# if file.endswith(".jpg") or file.endswith(".png"):
if file.endswith(".pts"):
png_file_arr.append(os.path.join(IbugConf.img_path_prefix, file[:-3] + "jpg"))
png_file_name.append(file)
number_of_samples = IbugConf.origin_number_of_all_sample
# number_of_samples = 1000
def test_per_image(self, counter, model, img, multitask, detect, org_img, x1, y1, scale_x, scale_y):
image_utility = ImageUtility()
pose_predicted = []
image = np.expand_dims(img, axis=0)
if multitask:
labels_main = np.swapaxes(model.predict(image)[0], 0, 1)
# noise_lbls = model.predict(image)[1]
# eyes_lbls = model.predict(image)[2]
# face_lbls = model.predict(image)[3]
# mouth_lbls = model.predict(image)[4]
pose_predicted = model.predict(image)[1][0]
# lbls_partial = np.swapaxes(np.concatenate((face_lbls, noise_lbls, eyes_lbls, mouth_lbls), axis=1), 0, 1) #
labels_predicted = labels_main
# labels_predicted = lbls_partial
# labels_predicted = (lbls_partial + labels_main) / 2.0
else:
labels_predicted = np.swapaxes(model.predict(image), 0, 1)
# labels_true_transformed, landmark_arr_x_t, landmark_arr_y_t = image_utility. \
# create_landmarks_from_normalized(labels_true, 224, 224, 112, 112)
#
# labels_predicted_asm = self.__post_process_correction(labels_predicted)
# labels_predict_asm_transformed, landmark_arr_x_asm_p, landmark_arr_y_asm_p = image_utility. \
# create_landmarks_from_normalized(labels_predicted_asm, 224, 224, 112, 112)
# imgpr.print_image_arr(counter+1, img, [], [])
# imgpr.print_image_arr(counter+1, img, landmark_arr_x_p, landmark_arr_y_p)
# imgpr.print_image_arr(counter+1, org_img, landmark_arr_x_p, landmark_arr_y_p)
# imgpr.print_image_arr((counter+1)*1000, img, landmark_arr_x_t, landmark_arr_y_t)
# imgpr.print_image_arr((counter+1)*100000, img, landmark_arr_x_asm_p, landmark_arr_y_asm_p)
'''pose estimation vs hopeNet'''
img_cp_1 = np.array(img) * 255.0
r, g, b = cv2.split(img_cp_1)
img_cp_1 = cv2.merge([b, g, r])
img_cp_2 = np.array(img) * 255.0
r, g, b = cv2.split(img_cp_2)
img_cp_2 = cv2.merge([b, g, r])
# yaw_truth, pitch_truth, roll_truth = 0, 0, 0
yaw_truth, pitch_truth, roll_truth = detect.detect(img, isFile=False, show=False)
yaw_p = pose_predicted[0]
pitch_p = pose_predicted[1]
roll_p = pose_predicted[2]
''' normalized to normal '''
min_degree = - 65
max_degree = 65
yaw_tpre = min_degree + (max_degree - min_degree) * (yaw_p + 1) / 2
pitch_tpre = min_degree + (max_degree - min_degree) * (pitch_p + 1) / 2
roll_tpre = min_degree + (max_degree - min_degree) * (roll_p + 1) / 2
output_pre = utils.draw_axis(org_img, yaw_tpre, pitch_tpre, roll_tpre, tdx=float(x1) + 112, tdy=float(y1) + 112,
size=112)
labels_predict_transformed, landmark_arr_x_p, landmark_arr_y_p, img_cv2 = image_utility. \
create_landmarks_from_normalized_original_img(output_pre, labels_predicted, 224, 224, 112, 112, float(x1),
float(y1), scale_x, scale_y)
# cv2.imwrite(str(counter) + ".jpg", output_pre, [int(cv2.IMWRITE_JPEG_QUALITY), 100])
mae_yaw = abs(yaw_tpre - yaw_truth)
mae_pitch = abs(pitch_tpre - pitch_truth)
mae_roll = abs(roll_tpre - roll_truth)
"================================="
return output_pre
augmentation_factor = 3 # create 100 image from 1
augmentation_factor_rotate = 20 # create 100 image from 1
sum_of_train_samples = origin_number_of_train_sample * augmentation_factor
sum_of_validation_samples = origin_number_of_evaluation_sample * augmentation_factor
img_path_prefix = '/home/soroush/PycharmProjects/Bodytracking/dataloader/LSP/lsp_dataset_original/test_images/'
pts_path_prefix = '/home/soroush/PycharmProjects/Bodytracking/dataloader/LSP/lsp_dataset_original/test_points/'
rotated_img_path_prefix = '/home/soroush/PycharmProjects/Bodytracking/dataloader/LSP/lsp_dataset_original/images_rotated/'
rotated_pts_path_prefix = '/home/soroush/PycharmProjects/Bodytracking/dataloader/LSP/lsp_dataset_original/points_rotated/'
before_heatmap_img_path_prefix = '/home/soroush/PycharmProjects/Bodytracking/dataloader/LSP/lsp_dataset_original/heatmap/'
image_utility = ImageUtility()
import random
png_file_arr = []
png_file_name = []
for file in sorted(os.listdir(IbugConf.img_path_prefix)):
if file.endswith(".jpg") or file.endswith(".png"):
png_file_arr.append(os.path.join(IbugConf.img_path_prefix, file))
png_file_name.append(file)
number_of_samples = IbugConf.origin_number_of_all_sample
number_of_samples = 1000
file_name_image = '/home/soroush/PycharmProjects/Bodytracking/dataloader/LSP/lsp_dataset_original/test_images_crop/'
file_name_points = '/home/soroush/PycharmProjects/Bodytracking/dataloader/LSP/lsp_dataset_original/test_images_crop/'
def _test_result_per_image(self, counter, model, img, labels_true):
image_utility = ImageUtility()
image = np.expand_dims(img*255, axis=0)
predict = model.predict(image)
pre_points = predict[0]
pose_predicted = [0, 0, 0]
# pre_points = pre_points.reshape([196])
labels_true_transformed, landmark_arr_x_t, landmark_arr_y_t = image_utility. \
create_landmarks_from_normalized(labels_true, 224, 224, 112, 112)
labels_predict_transformed, landmark_arr_x_p, landmark_arr_y_p = \
image_utility.create_landmarks_from_normalized(pre_points, 224, 224, 112, 112)
'''asm pp'''
# xy_h_p_asm = self.__post_process_correction(xy_h_p, True)
# labels_predict_transformed_asm, landmark_arr_x_p_asm, landmark_arr_y_p_asm = image_utility. \
# create_landmarks_from_normalized(xy_h_p_asm, 224, 224, 112, 112)
# labels_predict_transformed = labels_predict_transformed_asm
''''''
'''test print'''
# imgpr.print_image_arr(str((counter + 1))+'_img', img, [], [])
# imgpr.print_image_arr(str((counter + 1))+'_pred', img, landmark_arr_x_p, landmark_arr_y_p)
# imgpr.print_image_arr(str((counter + 1))+'_true', img, landmark_arr_x_t, landmark_arr_y_t)
# imgpr.print_image_arr((counter + 1), img, landmark_arr_x_t, landmark_arr_y_t)
# print("landmark_arr_x_t: " + str(landmark_arr_x_t))
# print("landmark_arr_x_p :" + str(landmark_arr_x_p))
#
# print("landmark_arr_y_t: " + str(landmark_arr_y_t))
# print("landmark_arr_y_p :" + str(landmark_arr_y_p))
# return 0, 0, 0, 0, 0, 0
# normalizing_distance = self.__calculate_interpupil_distance(labels_true_transformed)
normalizing_distance = self.__calculate_interoccular_distance(labels_true_transformed)
# return 1, 1, 1, 1, 1, 1
sum_errors = 0
for i in range(0, len(labels_true_transformed), 2): # two step each time
'''this is the value after transformation to the real points'''
x_point_predicted = labels_predict_transformed[i]
y_point_predicted = labels_predict_transformed[i + 1]
# x_point_predicted_asm = labels_predict_asm_transformed[i]
# y_point_predicted_asm = labels_predict_asm_transformed[i+1]
#
x_point_true = labels_true_transformed[i]
y_point_true = labels_true_transformed[i + 1]
'''this is the normalized value, which predicted by network'''
error = math.sqrt(((x_point_predicted - x_point_true) ** 2) + ((y_point_predicted - y_point_true) ** 2))
sum_errors += error
normalized_mean_error = sum_errors / (normalizing_distance * (self.num_landmark / 2))
# print(normalized_mean_error)
# print('=====')
lp = np.array(labels_predict_transformed).reshape([self.num_landmark // 2, 2])
lt = np.array(labels_true_transformed).reshape([self.num_landmark // 2, 2])
# print(labels_true_transformed)
# print(lt)
# print('---------------')
'''When there is no pose:'''
return normalized_mean_error, lt, lp, 0, 0, 0
def __customLoss_base(self, yTrue, yPred):
pca_utility = PCAUtility()
image_utility = ImageUtility()
tf_record_utility = TFRecordUtility()
eigenvalues, eigenvectors, meanvector = pca_utility.load_pca_obj(DatasetName.ibug)
# yTrue = tf.constant([[1.0, 2.0, 3.0], [5.0, 4.0, 7.0]])
# yPred = tf.constant([[9.0, 1.0, 2.0], [7.0, 3.0, 8.0]])
# session = K.get_session()
tensor_mean_square_error = K.mean(K.square(yPred - yTrue), axis=-1)
# tensor_mean_square_error = keras.losses.mean_squared_error(yPred, yTrue)
mse = K.eval(tensor_mean_square_error)
yPred_arr = K.eval(yPred)
yTrue_arr = K.eval(yTrue)
loss_array = []
for i in range(LearningConfig.batch_size):
asm_loss = 0
truth_vector = yTrue_arr[i]
predicted_vector = yPred_arr[i]
b_vector_p = self.calculate_b_vector(predicted_vector, True, eigenvalues, eigenvectors, meanvector)
y_pre_asm = meanvector + np.dot(eigenvectors, b_vector_p)
"""in order to test the results after PCA, you can use these lines of code"""
# landmark_arr_xy, landmark_arr_x, landmark_arr_y = image_utility.create_landmarks_from_normalized(truth_vector, 224, 224, 112, 112)
# image_utility.print_image_arr(i, np.ones([224, 224]), landmark_arr_x, landmark_arr_y)
#
# landmark_arr_xy_new, landmark_arr_x_new, landmark_arr_y_new= image_utility.create_landmarks_from_normalized(y_pre_asm, 224, 224, 112, 112)
# image_utility.print_image_arr(i*100, np.ones([224, 224]), landmark_arr_x_new, landmark_arr_y_new)
for j in range(len(y_pre_asm)):
asm_loss += (truth_vector[j] - y_pre_asm[j]) ** 2
asm_loss /= len(y_pre_asm)
# asm_loss *= mse[i]
# asm_loss *= LearningConfig.regularization_term
loss_array.append(asm_loss)
print('mse[i]' + str(mse[i]))
print('asm_loss[i]' + str(asm_loss))
print('============' )
loss_array = np.array(loss_array)
tensor_asm_loss = K.variable(loss_array)
# sum_loss_tensor = tf.math.add(tensor_mean_square_error, tensor_asm_loss)
tensor_total_loss = tf.reduce_mean([tensor_mean_square_error, tensor_asm_loss], axis=0)
# sum_loss = np.array(K.eval(tensor_asm_loss))
# print(mse)
# print(K.eval(tensor_mean_square_error))
# print(K.eval(tensor_asm_loss))
# print('asm_loss ' + str(loss_array[0]))
# print('mse_loss ' + str(mse[0]))
# print('sum_loss ' + str(sum_loss[0]))
# print('total_loss ' + str(total_loss[0]))
# print(' ')
return tensor_total_loss
keypoints_start = [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 2, 3]
keypoints_end = [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13, 8, 9]
# skeleton_color = [[255, 255, 255],[255, 0, 0],[0, 255, 0],
# [0, 0,255],[255, 255, 0], [0, 255, 255],
# [255, 0, 255],[192, 192, 192],[128, 128, 0],
# [128, 0, 0],[128, 128, 0],[0, 128,0],
# [128, 0, 128]]
skeleton_color = [[100, 100, 100],[101, 101, 101],[102, 102, 102],
[103, 103, 103],[104, 104, 104], [105, 105, 105],
[106, 106, 106],[107, 107, 107], [108, 108, 108],
[109, 109, 109],[110, 110, 110], [111, 111, 111],
[112, 112, 112]]
image_utility = ImageUtility()
for file_ in os.listdir(dir_heatmap):
if file_.endswith(".jpg"):
npy_file = file_[0:-4] + ".npy"
points_file = file_[0:-4] + ".pts"
points_arr = []
points_x_arr = []
points_y_arr = []
with open(dir_heatmap + points_file) as fp:
line = fp.readline()
cnt = 1
while line:
if 3 < cnt < 18: