def print_partial(counter, img, landmarks_arr): image_utility = ImageUtility() # plt.figure() # plt.imshow(img) # implot = plt.imshow(img) index =0 for lndm in landmarks_arr: image = Image.new("L", (InputDataSize.image_input_size // 4, InputDataSize.image_input_size // 4)) draw = ImageDraw.Draw(image) # color = "#"+''.join([random.choice('0123456789ABCDEF') for j in range(6)]) landmark_arr_xy, landmark_arr_x, landmark_arr_y = image_utility.create_landmarks_from_normalized(lndm, 224, 224, 112, 112) # plt.scatter(x=landmark_arr_x[:], y=landmark_arr_y[:], c=color, s=20) # plt.plot(landmark_arr_x, landmark_arr_y, '-ok', c=color) # if index == 4 or index == 5 or index == 6 or index == 7: # draw.polygon((landmark_arr_xy), fill='#ffffff') landmark_arr_xy = (np.array(landmark_arr_xy) // 4).tolist() draw.line((landmark_arr_xy), fill='#ffffff', width=2) img_np = np.asarray(image) plt.imshow(img_np) image.save('0_name_' + str(counter) + '_' + str(index) + '.png') index += 1
def _points_to_2d_face_graph(self, _points): """ :param points: [bs, 136] :return: [bs, 56, 56, num_fg] """ '''rescale points''' points = np.zeros([LearningConfig.batch_size, self.num_landmark]) image_utility = ImageUtility() indx = 0 for item in _points: point_scaled, px_1, py_1 = image_utility.create_landmarks_from_normalized( item, InputDataSize.hm_size, InputDataSize.hm_size, InputDataSize.hm_center, InputDataSize.hm_center) # imgpr.print_image_arr('pts_' + str(indx), np.zeros([56,56]), px_1, py_1) points[indx, :] = point_scaled indx += 1 '''create partial parts: ''' partial_points = self._slice_face_graph_np(points) '''convert from flatten to 2d''' points_2d = [] for pnt in partial_points: points_2d.append( pnt.reshape([LearningConfig.batch_size, len(pnt[1]) // 2, 2])) '''create the spare img for each facial part:''' hm_img = np.zeros([ LearningConfig.batch_size, InputDataSize.hm_size, InputDataSize.hm_size, self.num_face_graph_elements ]) # bs, 12 * 2 for i in range(LearningConfig.batch_size): for j in range(self.num_face_graph_elements): t_hm = np.zeros([InputDataSize.hm_size, InputDataSize.hm_size]) for x_y in points_2d[j][i]: if not (0 <= x_y[0] <= InputDataSize.hm_size - 1): x_y[0] = 0 if not (0 <= x_y[1] <= InputDataSize.hm_size - 1): x_y[1] = 0 t_hm[int(x_y[1]), int(x_y[0])] = 1 hm_img[i, :, :, j] = t_hm return hm_img
def _points_to_2d(self, _points): """ :param _points: :return: """ tf_rec = TFRecordUtility(self.num_landmark) image_utility = ImageUtility() hm_arr = [] for i in range(LearningConfig.batch_size): _x_y, _x, _y = image_utility.create_landmarks_from_normalized( _points[i], InputDataSize.image_input_size, InputDataSize.image_input_size, InputDataSize.img_center, InputDataSize.img_center) hm_multi_layer = tf_rec.generate_hm(InputDataSize.hm_size, InputDataSize.hm_size, np.array(_x_y), self.hm_stride / 2, False) hm = np.sum(hm_multi_layer, axis=2) hm_arr.append(hm) return np.array(hm_arr)
def print_hm_cord(self, epoch, step, images, heatmaps_gr, heatmaps_pr, points_gr, points_pr): epoch = 0 image_utility = ImageUtility() for i in range(LearningConfig.batch_size): # imgpr.print_image_arr_heat('hgr' + '.' + str(epoch)+"."+str(step)+'.'+str(i), heatmaps_gr[i]) imgpr.print_image_arr_heat( 'hpr' + '.' + str(epoch) + "." + str(step) + '.' + str(i), heatmaps_pr[i]) # landmark_arr_flat_n, landmark_arr_x_n, landmark_arr_y_n = image_utility.create_landmarks_from_normalized( # points_gr[i], 224, 224, 112, 112) # imgpr.print_image_arr('cgr' + '.' + str(epoch)+"."+str(step)+'.'+str(i), np.zeros([224, 224,3]), landmark_arr_x_n, # landmark_arr_y_n) landmark_arr_flat_n, landmark_arr_x_n, landmark_arr_y_n = image_utility.create_landmarks_from_normalized( points_pr[i], InputDataSize.image_input_size, InputDataSize.image_input_size, InputDataSize.image_input_size // 2, InputDataSize.image_input_size // 2) imgpr.print_image_arr( 'cpr' + '.' + str(epoch) + "." + str(step) + '.' + str(i), np.zeros([ InputDataSize.image_input_size, InputDataSize.image_input_size, 3 ]), landmark_arr_x_n, landmark_arr_y_n)
def _test_result_per_image(self, counter, model, img, labels_true): image_utility = ImageUtility() image = np.expand_dims(img*255, axis=0) predict = model.predict(image) pre_points = predict[0] pose_predicted = [0, 0, 0] # pre_points = pre_points.reshape([196]) labels_true_transformed, landmark_arr_x_t, landmark_arr_y_t = image_utility. \ create_landmarks_from_normalized(labels_true, 224, 224, 112, 112) labels_predict_transformed, landmark_arr_x_p, landmark_arr_y_p = \ image_utility.create_landmarks_from_normalized(pre_points, 224, 224, 112, 112) '''asm pp''' # xy_h_p_asm = self.__post_process_correction(xy_h_p, True) # labels_predict_transformed_asm, landmark_arr_x_p_asm, landmark_arr_y_p_asm = image_utility. \ # create_landmarks_from_normalized(xy_h_p_asm, 224, 224, 112, 112) # labels_predict_transformed = labels_predict_transformed_asm '''''' '''test print''' # imgpr.print_image_arr(str((counter + 1))+'_img', img, [], []) # imgpr.print_image_arr(str((counter + 1))+'_pred', img, landmark_arr_x_p, landmark_arr_y_p) # imgpr.print_image_arr(str((counter + 1))+'_true', img, landmark_arr_x_t, landmark_arr_y_t) # imgpr.print_image_arr((counter + 1), img, landmark_arr_x_t, landmark_arr_y_t) # print("landmark_arr_x_t: " + str(landmark_arr_x_t)) # print("landmark_arr_x_p :" + str(landmark_arr_x_p)) # # print("landmark_arr_y_t: " + str(landmark_arr_y_t)) # print("landmark_arr_y_p :" + str(landmark_arr_y_p)) # return 0, 0, 0, 0, 0, 0 # normalizing_distance = self.__calculate_interpupil_distance(labels_true_transformed) normalizing_distance = self.__calculate_interoccular_distance(labels_true_transformed) # return 1, 1, 1, 1, 1, 1 sum_errors = 0 for i in range(0, len(labels_true_transformed), 2): # two step each time '''this is the value after transformation to the real points''' x_point_predicted = labels_predict_transformed[i] y_point_predicted = labels_predict_transformed[i + 1] # x_point_predicted_asm = labels_predict_asm_transformed[i] # y_point_predicted_asm = labels_predict_asm_transformed[i+1] # x_point_true = labels_true_transformed[i] y_point_true = labels_true_transformed[i + 1] '''this is the normalized value, which predicted by network''' error = math.sqrt(((x_point_predicted - x_point_true) ** 2) + ((y_point_predicted - y_point_true) ** 2)) sum_errors += error normalized_mean_error = sum_errors / (normalizing_distance * (self.num_landmark / 2)) # print(normalized_mean_error) # print('=====') lp = np.array(labels_predict_transformed).reshape([self.num_landmark // 2, 2]) lt = np.array(labels_true_transformed).reshape([self.num_landmark // 2, 2]) # print(labels_true_transformed) # print(lt) # print('---------------') '''When there is no pose:''' return normalized_mean_error, lt, lp, 0, 0, 0
# roll_normalized = 2 * ((roll_predicted - min_degree) / (max_degree - min_degree)) - 1 # pose_array = np.array([yaw_normalized, pitch_normalized, roll_normalized]) pose_array = np.array([1, 1, 1]) '''normalize landmarks based on hyperface method''' width = len(resized_img_new[0]) height = len(resized_img_new[1]) x_center = width / 2 y_center = height / 2 landmark_arr_flat_normalized = [] for p in range(0, len(landmark_arr_flat), 2): landmark_arr_flat_normalized.append((x_center - landmark_arr_flat[p]) / width) landmark_arr_flat_normalized.append((y_center - landmark_arr_flat[p + 1]) / height) '''test print after augmentation''' landmark_arr_flat_n, landmark_arr_x_n, landmark_arr_y_n = image_utility.create_landmarks_from_normalized( landmark_arr_flat_normalized, 224, 224, 112, 112) # print_image_arr((i*100)+(k+1), resized_img_new, landmark_arr_x_n, landmark_arr_y_n) heatmap_landmark = generate_hm(56, 56, landmark_arr_flat_normalized, s=3.0) heatmap_landmark_all = np.sum(heatmap_landmark, axis=2) #print_image_arr(2*k, heatmap_landmark_all, [], []) # save heatmap file_name_save = png_file_name[i][0:-4] + "_" + str(k) + ".npy" hm_f = npy_dir + file_name_save # imgpr.print_image_arr_heat(1, hm, print_single=False) np.save(hm_f, heatmap_landmark) landmark_arr_flat_normalized = np.array(landmark_arr_flat_normalized)