def _create_generators(self): tf_utils = TFRecordUtility() if os.path.isfile('x_train_filenames.npy') and \ os.path.isfile('x_val_filenames.npy') and \ os.path.isfile('y_train_filenames.npy') and \ os.path.isfile('y_val_filenames.npy'): x_train_filenames = load('x_train_filenames.npy') x_val_filenames = load('x_val_filenames.npy') y_train = load('y_train_filenames.npy') y_val = load('y_val_filenames.npy') else: filenames, labels = tf_utils.create_image_and_labels_name() filenames_shuffled, y_labels_shuffled = shuffle(filenames, labels) x_train_filenames, x_val_filenames, y_train, y_val = train_test_split( filenames_shuffled, y_labels_shuffled, test_size=0.1, random_state=1) save('x_train_filenames.npy', x_train_filenames) save('x_val_filenames.npy', x_val_filenames) save('y_train_filenames.npy', y_train) save('y_val_filenames.npy', y_val) return x_train_filenames, x_val_filenames, y_train, y_val
def _create_generators(self): """ check if we have the img & lbls name. and create in case we need it. :return: """ fn_prefix = './file_names/' + self.dataset_name + '_' x_trains_path = fn_prefix + 'x_train_fns.npy' x_validations_path = fn_prefix + 'x_val_fns.npy' y_trains_path = fn_prefix + 'y_train_fns.npy' y_validations_path = fn_prefix + 'y_val_fns.npy' tf_utils = TFRecordUtility(number_of_landmark=self.num_landmark) if os.path.isfile(x_trains_path) and os.path.isfile(x_validations_path) \ and os.path.isfile(y_trains_path) and os.path.isfile(y_validations_path): x_train_filenames = load(x_trains_path) x_val_filenames = load(x_validations_path) y_train = load(y_trains_path) y_val = load(y_validations_path) else: filenames, labels = tf_utils.create_image_and_labels_name( dataset_name=self.dataset_name) filenames_shuffled, y_labels_shuffled = shuffle(filenames, labels) x_train_filenames, x_val_filenames, y_train, y_val = train_test_split( filenames_shuffled, y_labels_shuffled, test_size=0.05, random_state=1) save(x_trains_path, x_train_filenames) save(x_validations_path, x_val_filenames) save(y_trains_path, y_train) save(y_validations_path, y_val) return x_train_filenames, x_val_filenames, y_train, y_val
def create_pca(self, dataset_name, pca_postfix): tf_record_util = TFRecordUtility() lbl_arr = [] pose_arr = [] if dataset_name == DatasetName.ibug: lbl_arr, img_arr, pose_arr = tf_record_util.retrieve_tf_record( IbugConf.tf_train_path, IbugConf.sum_of_train_samples, only_label=True, only_pose=True) lbl_arr = np.array(lbl_arr) print('PCA-retrieved') '''need to be normalized based on the hyper face paper?''' # reduced_lbl_arr, eigenvalues, eigenvectors = self.__svd_func(lbl_arr, pca_postfix) reduced_lbl_arr, eigenvalues, eigenvectors = self.__func_PCA( lbl_arr, pca_postfix) mean_lbl_arr = np.mean(lbl_arr, axis=0) eigenvectors = eigenvectors.T self.__save_obj( eigenvalues, dataset_name + self.__eigenvalues_prefix + str(pca_postfix)) self.__save_obj( eigenvectors, dataset_name + self.__eigenvectors_prefix + str(pca_postfix)) self.__save_obj( mean_lbl_arr, dataset_name + self.__meanvector_prefix + str(pca_postfix)) '''calculate pose min max''' p_1_arr = [] p_2_arr = [] p_3_arr = [] for p_item in pose_arr: p_1_arr.append(p_item[0]) p_2_arr.append(p_item[1]) p_3_arr.append(p_item[2]) p_1_min = min(p_1_arr) p_1_max = max(p_1_arr) p_2_min = min(p_2_arr) p_2_max = max(p_2_arr) p_3_min = min(p_3_arr) p_3_max = max(p_3_arr) self.__save_obj(p_1_min, 'p_1_min') self.__save_obj(p_1_max, 'p_1_max') self.__save_obj(p_2_min, 'p_2_min') self.__save_obj(p_2_max, 'p_2_max') self.__save_obj(p_3_min, 'p_3_min') self.__save_obj(p_3_max, 'p_3_max') print('PCA-->done')
def _create_generators(self): fn_prefix = './file_names/' + self.dataset_name + '_' # x_trains_path = fn_prefix + 'x_train_fns.npy' # x_validations_path = fn_prefix + 'x_val_fns.npy' tf_utils = TFRecordUtility(number_of_landmark=self.num_landmark) filenames, labels = tf_utils.create_image_and_labels_name( img_path=self.img_path, annotation_path=self.annotation_path) filenames_shuffled, y_labels_shuffled = shuffle(filenames, labels) x_train_filenames, x_val_filenames, y_train, y_val = train_test_split( filenames_shuffled, y_labels_shuffled, test_size=LearningConfig.batch_size, random_state=1) # save(x_trains_path, filenames_shuffled) # save(x_validations_path, y_labels_shuffled) # save(x_trains_path, x_train_filenames) # save(x_validations_path, x_val_filenames) # save(y_trains_path, y_train) # save(y_validations_path, y_val) # return filenames_shuffled, y_labels_shuffled return x_train_filenames, x_val_filenames, y_train, y_val
def _eval_model(self, model, img_val_filenames, pn_val_filenames): tf_util = TFRecordUtility(number_of_landmark=self.num_landmark) nme_sum = 0 fail_counter_sum = 0 batch_size = 20 # LearningConfig.batch_size step_per_epoch = int(len(img_val_filenames) // (batch_size)) for batch_index in tqdm(range(step_per_epoch)): images, anno_gts = self._get_batch_sample( batch_index=batch_index, x_train_filenames=img_val_filenames, y_train_filenames=pn_val_filenames, is_eval=True, batch_size=batch_size) '''predict:''' anno_Pres = model.predict_on_batch( images) # hm_pr: 4, bs, 64, 64, 68 '''calculate NME for batch''' bath_nme, bath_fr = tf_util.calc_NME_over_batch( anno_GTs=anno_gts, anno_Pres=anno_Pres, ds_name=self.dataset_name) nme_sum += bath_nme fail_counter_sum += bath_fr '''calculate total''' fr = 100 * fail_counter_sum / len(img_val_filenames) nme = 100 * nme_sum / len(img_val_filenames) print('nme:' + str(nme)) print('fr:' + str(fr)) return nme, fr
def test_pca_validity(self, pca_postfix): cnn_model = CNNModel() pca_utility = PCAUtility() tf_record_utility = TFRecordUtility() image_utility = ImageUtility() eigenvalues, eigenvectors, meanvector = pca_utility.load_pca_obj( dataset_name=DatasetName.ibug, pca_postfix=pca_postfix) lbl_arr, img_arr, pose_arr = tf_record_utility.retrieve_tf_record( tfrecord_filename=IbugConf.tf_train_path, number_of_records=30, only_label=False) for i in range(20): b_vector_p = self.calculate_b_vector(lbl_arr[i], True, eigenvalues, eigenvectors, meanvector) lbl_new = meanvector + np.dot(eigenvectors, b_vector_p) labels_true_transformed, landmark_arr_x_t, landmark_arr_y_t = image_utility. \ create_landmarks_from_normalized(lbl_arr[i], 224, 224, 112, 112) labels_true_transformed_pca, landmark_arr_x_pca, landmark_arr_y_pca = image_utility. \ create_landmarks_from_normalized(lbl_new, 224, 224, 112, 112) image_utility.print_image_arr(i, img_arr[i], landmark_arr_x_t, landmark_arr_y_t) image_utility.print_image_arr(i * 1000, img_arr[i], landmark_arr_x_pca, landmark_arr_y_pca)
def _create_generators(self): tf_utils = TFRecordUtility() if self.point_wise: if True or os.path.isfile('npy/' +'x_train_filenames_pw.npy') and \ os.path.isfile('npy/' +'x_val_filenames_pw.npy') and \ os.path.isfile('npy/' +'y_train_filenames_pw.npy') and \ os.path.isfile('npy/' +'y_val_filenames_pw.npy'): x_train_filenames = load('npy/x_train_filenames_pw.npy') x_val_filenames = load('npy/x_val_filenames_pw.npy') y_train = load('npy/y_train_filenames_pw.npy') y_val = load('npy/y_val_filenames_pw.npy') else: for i in range(68): filenames, labels = tf_utils.create_fused_images_and_labels_name( ) filenames_shuffled, y_labels_shuffled = shuffle( filenames, labels) x_train_filenames, x_val_filenames, y_train, y_val = train_test_split( filenames_shuffled, y_labels_shuffled, test_size=0.1, random_state=1) save('npy/' + 'x_train_filenames_pw_.npy', x_train_filenames) save('npy/' + 'x_val_filenames_pw_.npy', x_val_filenames) save('npy/' + 'y_train_filenames_pw_.npy', y_train) save('npy/' + 'y_val_filenames_pw_.npy', y_val) else: if os.path.isfile('x_train_filenames.npy') and \ os.path.isfile('x_val_filenames.npy') and \ os.path.isfile('y_train_filenames.npy') and \ os.path.isfile('y_val_filenames.npy'): x_train_filenames = load('x_train_filenames.npy') x_val_filenames = load('x_val_filenames.npy') y_train = load('y_train_filenames.npy') y_val = load('y_val_filenames.npy') else: filenames, labels = tf_utils.create_image_and_labels_name( self.point_wise) filenames_shuffled, y_labels_shuffled = shuffle( filenames, labels) x_train_filenames, x_val_filenames, y_train, y_val = train_test_split( filenames_shuffled, y_labels_shuffled, test_size=0.1, random_state=1) save('x_train_filenames.npy', x_train_filenames) save('x_val_filenames.npy', x_val_filenames) save('y_train_filenames.npy', y_train) save('y_val_filenames.npy', y_val) return x_train_filenames, x_val_filenames, y_train, y_val
def _test_on_W300(self, model): tf_record_utility = TFRecordUtility(self.num_landmark) lbl_arr_challenging, img_arr_challenging = tf_record_utility.retrieve_tf_record_test_set( tfrecord_filename=IbugConf.tf_test_path_challenging, number_of_records=IbugConf.orig_number_of_test_challenging, only_label=False) nme_ch, fr_ch, auc_ch, mae_yaw_ch, mae_pitch_ch, mae_roll_ch = self._calculate_errors(model, W300Conf.number_of_all_sample_challenging, img_arr_challenging, lbl_arr_challenging) lbl_arr_challenging = np.array(lbl_arr_challenging) img_arr_challenging = np.array(img_arr_challenging) print('nme_ch: ', str(nme_ch), 'fr_ch: ', str(fr_ch), 'auc_ch: ', str(auc_ch)) print('mae_yaw: ', str(mae_yaw_ch), 'mae_pitch: ', str(mae_pitch_ch), 'mae_roll: ', str(mae_roll_ch)) lbl_arr_common, img_arr_common = tf_record_utility.retrieve_tf_record_test_set( tfrecord_filename=IbugConf.tf_test_path_common, number_of_records=IbugConf.orig_number_of_test_common, only_label=False) lbl_arr_full, img_arr_full = tf_record_utility.retrieve_tf_record_test_set( tfrecord_filename=IbugConf.tf_test_path_full, number_of_records=IbugConf.orig_number_of_test_full, only_label=False) lbl_arr_common = np.array(lbl_arr_common) img_arr_common = np.array(img_arr_common) lbl_arr_full = np.array(lbl_arr_full) img_arr_full = np.array(img_arr_full) nme_c, fr_c, auc_c, mae_yaw_c, mae_pitch_c, mae_roll_c = self._calculate_errors(model, W300Conf.number_of_all_sample_common, img_arr_common, lbl_arr_common) print('nme_c: ', str(nme_c), 'fr_c: ', str(fr_c), 'auc_c: ', str(auc_c)) print('mae_yaw: ', str(mae_yaw_c), 'mae_pitch: ', str(mae_pitch_c), 'mae_roll: ', str(mae_roll_c)) nme_f, fr_f, auc_f, mae_yaw_f, mae_pitch_f, mae_roll_f = self._calculate_errors(model, W300Conf.number_of_all_sample_full, img_arr_full, lbl_arr_full) print('nme_f: ', str(nme_f), 'fr_f: ', str(fr_f), 'auc_f: ', str(auc_f)) print('mae_yaw: ', str(mae_yaw_f), 'mae_pitch: ', str(mae_pitch_f), 'mae_roll: ', str(mae_roll_f)) result_str = "-------------------------_test_on_W300------------------------------------" + '\n\r' \ 'nme_ch: ', str(nme_ch), 'fr_ch: ', str(fr_ch), 'auc_ch: ', str(auc_ch) + '\n\r' \ 'mae_yaw: ', str(mae_yaw_ch), 'mae_pitch: ', str(mae_pitch_ch), 'mae_roll: ', str(mae_roll_ch) + '\n\r' \ 'nme_c: ', str(nme_c), 'fr_c: ', str(fr_c), 'auc_c: ', str(auc_c) + '\n\r' \ 'mae_yaw: ', str(mae_yaw_c), 'mae_pitch: ', str(mae_pitch_c), 'mae_roll: ', str(mae_roll_c) + '\n\r' \ 'nme_f: ', str(nme_f), 'fr_f: ', str(fr_f), 'auc_f: ', str(auc_f) + '\n\r' \ 'mae_yaw: ', str(mae_yaw_f), 'mae_pitch: ', str(mae_pitch_f), 'mae_roll: ', str(mae_roll_f) return str(result_str)
def _test_on_COFW(self, model): tf_record_utility = TFRecordUtility(self.output_len) lbl_arr_total, img_arr_total = tf_record_utility.retrieve_tf_record_test_set( tfrecord_filename=CofwConf.tf_test_path, number_of_records=CofwConf.orig_number_of_test, only_label=False) lbl_arr_total = np.array(lbl_arr_total) img_arr_total = np.array(img_arr_total) nme, fr, auc, = self._calculate_errors(model, CofwConf.orig_number_of_test, img_arr_total, lbl_arr_total) print('nme: ', str(nme), 'fr: ', str(fr), 'auc: ', str(auc)) print("-------------------------------------------------------------")
def _create_generators(self, img_path=None, annotation_path=None): # x_trains_path = fn_prefix + 'x_train_fns.npy' # x_validations_path = fn_prefix + 'x_val_fns.npy' tf_utils = TFRecordUtility(number_of_landmark=self.num_landmark) if img_path is None: filenames, labels = tf_utils.create_image_and_labels_name( img_path=self.img_path, annotation_path=self.annotation_path) else: filenames, labels = tf_utils.create_image_and_labels_name( img_path=img_path, annotation_path=annotation_path) filenames_shuffled, y_labels_shuffled = shuffle(filenames, labels) # x_train_filenames, x_val_filenames, y_train, y_val = train_test_split( # filenames_shuffled, y_labels_shuffled, test_size=LearningConfig.batch_size, random_state=1) return filenames_shuffled, y_labels_shuffled
def _create_generators(self): tf_utils = TFRecordUtility(self.output_len) filenames, labels = tf_utils.create_image_and_labels_name() filenames_shuffled, y_labels_shuffled = shuffle(filenames, labels) x_train_filenames, x_val_filenames, y_train, y_val = train_test_split( filenames_shuffled, y_labels_shuffled, test_size=0.05, random_state=100, shuffle=True) save('x_train_filenames.npy', x_train_filenames) save('x_val_filenames.npy', x_val_filenames) save('y_train_filenames.npy', y_train) save('y_val_filenames.npy', y_val) return x_train_filenames, x_val_filenames, y_train, y_val
def convert_pts_to_hm(self, pts): """ we want to convert pts to hm{56 * 56 * self.num_landmark//2} and then calculate loss :param pts: bs * self.num_landmark: pts are normal between -0.5, +0.5 --> need to upsample :return: """ pts = pts.numpy() # pts = tf.keras.backend.eval(pts) hm_arr = [] tf_util = TFRecordUtility(self.num_landmark // 2) for i in range(LearningConfig.batch_size): hm_t = tf_util.generate_hm_Ten(height=InputDataSize.hm_size, width=InputDataSize.hm_size, landmarks=pts[i], s=3.0, upsample=True) hm_arr.append(hm_t) hm_pts = tf.convert_to_tensor(np.array(hm_arr)) return hm_pts
def train_fit(self): tf_record_util = TFRecordUtility(self.output_len) '''prepare callbacks''' callbacks_list = self._prepare_callback() ''' define optimizers''' optimizer = self._get_optimizer() '''create train, validation, test data iterator''' train_images, train_landmarks = tf_record_util.create_training_tensor_points( tfrecord_filename=self.tf_train_path, batch_size=self.BATCH_SIZE) validation_images, validation_landmarks = \ tf_record_util.create_training_tensor_points(tfrecord_filename=self.tf_eval_path, batch_size=self.BATCH_SIZE) '''creating model''' cnn = CNNModel() # model = tf.keras.models.load_model(self.weight) model = cnn.get_model(train_images=train_images, arch=self.arch, num_output_layers=self.num_output_layers, output_len=self.output_len, input_tensor=train_images, inp_shape=None) if self.weight is not None: model.load_weights(self.weight) '''compiling model''' model.compile( loss=self._generate_loss(), optimizer=optimizer, metrics=['mse', 'mae'], target_tensors=self._generate_target_tensors(train_landmarks), loss_weights=self._generate_loss_weights()) '''train Model ''' print('< ========== Start Training ============= >') history = model.fit(train_images, train_landmarks, epochs=self.EPOCHS, steps_per_epoch=self.STEPS_PER_EPOCH, validation_data=(validation_images, validation_landmarks), validation_steps=self.STEPS_PER_VALIDATION_EPOCH, verbose=1, callbacks=callbacks_list)
def _test_on_W300(self, model): tf_record_utility = TFRecordUtility(self.output_len) lbl_arr_challenging, img_arr_challenging = tf_record_utility.retrieve_tf_record_test_set( tfrecord_filename=W300Conf.tf_challenging, number_of_records=W300Conf.number_of_all_sample_challenging, only_label=False) lbl_arr_common, img_arr_common = tf_record_utility.retrieve_tf_record_test_set( tfrecord_filename=W300Conf.tf_common, number_of_records=W300Conf.number_of_all_sample_common, only_label=False) lbl_arr_full, img_arr_full = tf_record_utility.retrieve_tf_record_test_set( tfrecord_filename=W300Conf.tf_full, number_of_records=W300Conf.number_of_all_sample_full, only_label=False) lbl_arr_challenging = np.array(lbl_arr_challenging) img_arr_challenging = np.array(img_arr_challenging) lbl_arr_common = np.array(lbl_arr_common) img_arr_common = np.array(img_arr_common) lbl_arr_full = np.array(lbl_arr_full) img_arr_full = np.array(img_arr_full) nme_ch, fr_ch, auc_ch = self._calculate_errors( model, W300Conf.number_of_all_sample_challenging, img_arr_challenging, lbl_arr_challenging) print('nme_ch: ', str(nme_ch), 'fr_ch: ', str(fr_ch), 'auc_ch: ', str(auc_ch)) nme_c, fr_c, auc_c = self._calculate_errors( model, W300Conf.number_of_all_sample_common, img_arr_common, lbl_arr_common) print('nme_c: ', str(nme_c), 'fr_c: ', str(fr_c), 'auc_c: ', str(auc_c)) nme_f, fr_f, auc_f = self._calculate_errors( model, W300Conf.number_of_all_sample_full, img_arr_full, lbl_arr_full) print('nme_f: ', str(nme_f), 'fr_f: ', str(fr_f), 'auc_f: ', str(auc_f))
def _points_to_2d(self, _points): """ :param _points: :return: """ tf_rec = TFRecordUtility(self.num_landmark) image_utility = ImageUtility() hm_arr = [] for i in range(LearningConfig.batch_size): _x_y, _x, _y = image_utility.create_landmarks_from_normalized( _points[i], InputDataSize.image_input_size, InputDataSize.image_input_size, InputDataSize.img_center, InputDataSize.img_center) hm_multi_layer = tf_rec.generate_hm(InputDataSize.hm_size, InputDataSize.hm_size, np.array(_x_y), self.hm_stride / 2, False) hm = np.sum(hm_multi_layer, axis=2) hm_arr.append(hm) return np.array(hm_arr)
def convert_hm_to_pts(self, hm): x_center = InputDataSize.image_input_size // 2 width = InputDataSize.image_input_size hm_arr = [] tf_util = TFRecordUtility(self.num_landmark // 2) for i in range(LearningConfig.batch_size): hm_t = tf_util.from_heatmap_to_point_tensor(heatmaps=hm[i], number_of_points=5) hm_t = tf.reshape(tensor=hm_t, shape=self.num_landmark) '''hm is in [0,224] --> should be in [-0.5,+0.5]''' hm_t_norm = tf.math.scalar_mul( scalar=1 / width, x=tf.math.subtract(hm_t, np.repeat(x_center, self.num_landmark))) hm_arr.append(hm_t_norm) '''reshape hm''' hm_pts = tf.stack( [hm_arr[i] for i in range(LearningConfig.batch_size)], 0) # bs * self.num_landmark return hm_pts
def custom_loss_hm(self, ten_hm_t, ten_hm_p): # print(ten_hm_t.get_shape()) # [None, 56, 56, 68] # print(ten_hm_p.get_shape()) tf_utility = TFRecordUtility() sqr = K.square(ten_hm_t - ten_hm_p) # [None, 56, 56, 68] mean1 = K.mean(sqr, axis=1) mean2 = K.mean(mean1, axis=1) tensor_mean_square_error = K.mean(mean2, axis=1) # print(tensor_mean_square_error.get_shape().as_list()) # [None, 68] # vec_mse = K.eval(tensor_mean_square_error) # print("mse.shape:") # print(vec_mse.shape) # (50, 68) # print(vec_mse) # print("----------->>>") '''calculate points from generated hm''' p_points_batch = tf.stack([ tf_utility.from_heatmap_to_point_tensor(ten_hm_p[i], 5, 1) for i in range(LearningConfig.batch_size) ]) t_points_batch = tf.stack([ tf_utility.from_heatmap_to_point_tensor(ten_hm_t[i], 5, 1) for i in range(LearningConfig.batch_size) ]) '''p_points_batch is [batch, 2, 68]''' sqr_2 = K.square(t_points_batch - p_points_batch) # [None, 2, 68] mean_1 = K.mean(sqr_2, axis=1) tensor_indices_mean_square_error = K.mean(mean_1, axis=1) # tensor_total_loss = tf.reduce_mean([tensor_mean_square_error, tensor_indices_mean_square_error]) tensor_total_loss = tf.add(tensor_mean_square_error, tensor_indices_mean_square_error) return tensor_total_loss
def _test_on_COFW(self, model): tf_record_utility = TFRecordUtility(self.output_len) lbl_arr_total, img_arr_total = tf_record_utility.retrieve_tf_record_test_set( tfrecord_filename=CofwConf.tf_test_path, number_of_records=CofwConf.orig_number_of_test, only_label=False) lbl_arr_total = np.array(lbl_arr_total) img_arr_total = np.array(img_arr_total) nme, fr, auc, mae_yaw, mae_pitch, mae_roll = self._calculate_errors(model, CofwConf.orig_number_of_test, img_arr_total, lbl_arr_total) result_str = "-------------------------_test_on_COFW------------------------------------" + '\n\r' \ + 'nme: ', str(nme), 'fr: ', str(fr), 'auc: ', str(auc) + '\n\r' \ + 'mae_yaw: ', str(mae_yaw), 'mae_pitch: ', str(mae_pitch), 'mae_roll: ', str(mae_roll) print('nme: ', str(nme), 'fr: ', str(fr), 'auc: ', str(auc)) print('mae_yaw: ', str(mae_yaw), 'mae_pitch: ', str(mae_pitch), 'mae_roll: ', str(mae_roll)) print("-------------------------------------------------------------") return result_str
def train_fit(self): tf_record_util = TFRecordUtility() '''prepare callbacks''' callbacks_list = self._prepare_callback() ''' define optimizers''' optimizer = self._get_optimizer() '''create train, validation, test data iterator''' train_images, _, _, _, _, _, _, train_heatmap, _ = \ tf_record_util.create_training_tensor(tfrecord_filename=IbugConf.tf_train_path_heatmap, batch_size=self.BATCH_SIZE, reduced=True) validation_images, _, _, _, _, _, _, validation_heatmap, _ = \ tf_record_util.create_training_tensor(tfrecord_filename=IbugConf.tf_evaluation_path_heatmap, batch_size=self.BATCH_SIZE, reduced=True) '''creating model''' model = self._get_model(train_images) if self.weight is not None: model.load_weights(self.weight) '''compiling model''' model.compile( loss=self._generate_loss(), optimizer=optimizer, metrics=['mse', 'mae'], target_tensors=self._generate_target_tensors(train_heatmap), loss_weights=self._generate_loss_weights()) '''train Model ''' print('< ========== Start Training ============= >') history = model.fit( train_images, train_heatmap, epochs=self.EPOCHS, steps_per_epoch=self.STEPS_PER_EPOCH, validation_data=(validation_images, validation_heatmap), validation_steps=self.STEPS_PER_VALIDATION_EPOCH, verbose=1, callbacks=callbacks_list, )
def calculate_adoptive_weight(self, epoch, batch_index, y_train_filenames): tf_utils = TFRecordUtility(self.num_landmark) batch_y = y_train_filenames[batch_index * LearningConfig.batch_size:(batch_index + 1) * LearningConfig.batch_size] if 0 <= epoch <= 10: asm_acc = 80 elif 10 < epoch <= 20: asm_acc = 85 elif 20 < epoch <= 50: asm_acc = 90 elif 50 < epoch <= 100: asm_acc = 95 else: asm_acc = 97 asm_acc = 97 '''for each point in training set, calc delta_i = ASM(gt_i)-pr_i: ''' if self.dataset_name == DatasetName.cofw: # this ds is not normalized if self.arch == 'efNb0': pn_batch = np.array([ self._load_and_normalize(self.annotation_path + file_name) for file_name in batch_y ]) pn_batch_asm = np.array([ tf_utils.get_asm( input=self._load_and_normalize(self.annotation_path + file_name), dataset_name=self.dataset_name, accuracy=asm_acc) for file_name in batch_y ]) else: pn_batch = np.array([ load(self.annotation_path + file_name) for file_name in batch_y ]) pn_batch_asm = np.array([ tf_utils.get_asm(input=load(self.annotation_path + file_name), dataset_name=self.dataset_name, accuracy=asm_acc) for file_name in batch_y ]) else: pn_batch = np.array([ self._load_and_normalize(self.annotation_path + file_name) for file_name in batch_y ]) pn_batch_asm = np.array([ tf_utils.get_asm( input=self._load_and_normalize(self.annotation_path + file_name), dataset_name=self.dataset_name, accuracy=asm_acc) for file_name in batch_y ]) '''phi = mean(delta_i s)''' '''1d''' delta = np.array([ abs(pn_batch[i] - pn_batch_asm[i]) for i in range(len(pn_batch)) ]) # 1-d '''2d''' # pn_batch = np.reshape(pn_batch, [pn_batch.shape[0], pn_batch.shape[1] // 2, 2]) # bs, 68, 2 # pn_batch_asm = np.reshape(pn_batch_asm, [pn_batch_asm.shape[0], pn_batch_asm.shape[1] // 2, 2]) # bs, 68, 2 # delta = np.sqrt(np.sum(np.square(pn_batch - pn_batch_asm), axis=2)) omega = np.zeros_like(delta) for i in range(len(delta)): omega[i] = np.array([ 1 / (0.2 + math.exp(-50 * delta[i][j])) for j in range(len(delta[i])) ]) return omega
def _test_on_WFLW(self, model): tf_record_utility = TFRecordUtility(self.num_landmark) result_str = '' # lbl_arr_total, img_arr_total = tf_record_utility.retrieve_tf_record_test_set( # tfrecord_filename=WflwConf.tf_test_path, # number_of_records=WflwConf.orig_number_of_test, # only_label=False) # nme, fr, auc, mae_yaw, mae_pitch, mae_roll = self._calculate_errors(detect, model, WflwConf.orig_number_of_test, # img_arr_total, lbl_arr_total) # result_str += "\n\r\n\r-------------------------_test_on_WFLW:total------------------------------------" + '\n\r' \ # + 'nme: ', str(nme), 'fr: ', str(fr), 'auc: ', str(auc) + '\n\r' \ # + 'mae_yaw: ', str(mae_yaw), 'mae_pitch: ', str(mae_pitch), 'mae_roll: ', str(mae_roll) # # print('nme: ', str(nme), 'fr: ', str(fr), 'auc: ', str(auc)) # print('mae_yaw: ', str(mae_yaw), 'mae_pitch: ', str(mae_pitch), 'mae_roll: ', str(mae_roll)) # print("-------------------------------------------------------------") # lbl_arr_largepose, img_arr_largepose = tf_record_utility.retrieve_tf_record_test_set( tfrecord_filename=WflwConf.tf_test_path_largepose, number_of_records=WflwConf.orig_of_all_test_largepose, only_label=False) nme, fr, auc, mae_yaw, mae_pitch, mae_roll = self._calculate_errors(model, WflwConf.orig_of_all_test_largepose, img_arr_largepose, lbl_arr_largepose) result_str += '\n\r\n\r-------------------------_test_on_WFLW: largepose------------------------------------nme: '\ + str(nme)+'fr: ' + str(fr) + 'auc: '+ str(auc) + '\n\r' + 'mae_yaw: '+ str(mae_yaw)+ 'mae_pitch: '\ + str(mae_pitch)+ 'mae_roll: '+ str(mae_roll) print('nme: '+ str(nme)+ 'fr: ' + str(fr)+ 'auc: '+ str(auc)) print('mae_yaw: '+ str(mae_yaw)+ 'mae_pitch: '+ str(mae_pitch)+ 'mae_roll: '+ str(mae_roll)) print("-------------------------------------------------------------") lbl_arr_occlusion, img_arr_occlusion = tf_record_utility.retrieve_tf_record_test_set( tfrecord_filename=WflwConf.tf_test_path_occlusion, number_of_records=WflwConf.orig_of_all_test_occlusion, only_label=False) nme, fr, auc, mae_yaw, mae_pitch, mae_roll = self._calculate_errors(model, WflwConf.orig_of_all_test_occlusion, img_arr_occlusion, lbl_arr_occlusion) result_str += '\n\r\n\r-------------------------_test_on_WFLW _occlusion------------------------------------'+ '\n\r' \ + 'nme: '+ str(nme)+ 'fr: '+ str(fr)+ 'auc: '+ str(auc) + '\n\r' \ + 'mae_yaw: '+ str(mae_yaw)+ 'mae_pitch: '+ str(mae_pitch)+ 'mae_roll: '+ str(mae_roll) print('nme: '+ str(nme)+ 'fr: '+ str(fr)+ 'auc: '+ str(auc)) print('mae_yaw: '+ str(mae_yaw)+ 'mae_pitch: '+ str(mae_pitch)+ 'mae_roll: '+ str(mae_roll)) print("-------------------------------------------------------------") lbl_arr_makeup, img_arr_makeup = tf_record_utility.retrieve_tf_record_test_set( tfrecord_filename=WflwConf.tf_test_path_makeup, number_of_records=WflwConf.orig_of_all_test_makeup, only_label=False) nme, fr, auc, mae_yaw, mae_pitch, mae_roll = self._calculate_errors(model, WflwConf.orig_of_all_test_makeup, img_arr_makeup, lbl_arr_makeup) result_str += '\n\r\n\r-------------------------_test_on_WFLW_makeup------------------------------------' + '\n\r' \ + 'nme: '+ str(nme)+ 'fr: '+ str(fr)+ 'auc: '+ str(auc) + '\n\r' \ + 'mae_yaw: '+ str(mae_yaw)+ 'mae_pitch: '+ str(mae_pitch)+ 'mae_roll: '+ str(mae_roll) print('nme: '+ str(nme)+ 'fr: '+ str(fr)+ 'auc: '+ str(auc)) print('mae_yaw: '+ str(mae_yaw)+ 'mae_pitch: '+ str(mae_pitch)+ 'mae_roll: '+ str(mae_roll)) print("-------------------------------------------------------------") lbl_arr_expression, img_arr_expression = tf_record_utility.retrieve_tf_record_test_set( tfrecord_filename=WflwConf.tf_test_path_expression, number_of_records=WflwConf.orig_of_all_test_expression, only_label=False) nme, fr, auc, mae_yaw, mae_pitch, mae_roll = self._calculate_errors(model, WflwConf.orig_of_all_test_expression, img_arr_expression, lbl_arr_expression) result_str += '\n\r\n\r-------------------------_test_on_WFLW_expression------------------------------------' + '\n\r' \ + 'nme: '+ str(nme)+ 'fr: '+ str(fr)+ 'auc: '+ str(auc) + '\n\r' \ + 'mae_yaw: '+ str(mae_yaw)+ 'mae_pitch: '+ str(mae_pitch)+ 'mae_roll: '+ str(mae_roll) print('nme: '+ str(nme)+ 'fr: '+ str(fr)+ 'auc: '+ str(auc)) print('mae_yaw: '+ str(mae_yaw)+ 'mae_pitch: '+ str(mae_pitch)+ 'mae_roll: '+ str(mae_roll)) print("-------------------------------------------------------------") lbl_arr_illumination, img_arr_illumination = tf_record_utility.retrieve_tf_record_test_set( tfrecord_filename=WflwConf.tf_test_path_illumination, number_of_records=WflwConf.orig_of_all_test_illumination, only_label=False) nme, fr, auc, mae_yaw, mae_pitch, mae_roll = self._calculate_errors(model, WflwConf.orig_of_all_test_illumination, img_arr_illumination, lbl_arr_illumination) result_str += '\n\r\n\r-------------------------_test_on_WFLW _illumination ------------------------------------' + '\n\r' \ + 'nme: '+ str(nme)+ 'fr: '+ str(fr)+ 'auc: '+ str(auc) + '\n\r' \ + 'mae_yaw: '+ str(mae_yaw)+ 'mae_pitch: '+ str(mae_pitch)+ 'mae_roll: '+ str(mae_roll) print('nme: '+ str(nme)+ 'fr: '+ str(fr)+ 'auc: '+ str(auc)) print('mae_yaw: '+ str(mae_yaw)+ 'mae_pitch: '+ str(mae_pitch) + 'mae_roll: ' + str(mae_roll)) print("-------------------------------------------------------------") lbl_arr_blur, img_arr_blur = tf_record_utility.retrieve_tf_record_test_set( tfrecord_filename=WflwConf.tf_test_path_blur, number_of_records=WflwConf.orig_of_all_test_blur, only_label=False) nme, fr, auc, mae_yaw, mae_pitch, mae_roll = self._calculate_errors(model, WflwConf.orig_of_all_test_blur, img_arr_blur, lbl_arr_blur) result_str += '\n\r\n\r-------------------------_test_on_WFLW_blur------------------------------------' + '\n\r' \ + 'nme: '+ str(nme)+ 'fr: '+ str(fr)+ 'auc: '+ str(auc) + '\n\r' \ + 'mae_yaw: '+ str(mae_yaw)+ 'mae_pitch: '+ str(mae_pitch)+ 'mae_roll: '+ str(mae_roll) print('nme: '+ str(nme)+ 'fr: '+ str(fr)+ 'auc: '+ str(auc)) print('mae_yaw: '+ str(mae_yaw)+ 'mae_pitch: '+ str(mae_pitch)+ 'mae_roll: '+ str(mae_roll)) print("-------------------------------------------------------------") return result_str
def custom_loss_hm(self, ten_hm_t, ten_hm_p): # print(ten_hm_t.get_shape().as_list()) # [None, 56, 56, 68] # print(ten_hm_p.get_shape()) tf_utility = TFRecordUtility() sqr = K.square(ten_hm_t - ten_hm_p) # [None, 56, 56, 68] mean1 = K.mean(sqr, axis=1) mean2 = K.mean(mean1, axis=1) tensor_mean_square_error = K.mean(mean2, axis=1) # print(tensor_mean_square_error.get_shape().as_list()) # [None, 68] # vec_mse = K.eval(tensor_mean_square_error) # print("mse.shape:") # print(vec_mse.shape) # (50, 68) # print(vec_mse) # print("----------->>>") '''convert tensor to vector''' vec_hm_p = K.eval(ten_hm_p) vec_hm_t = K.eval(ten_hm_t) loss_array = [] for i in range(LearningConfig.batch_size): '''convert heatmap to points''' x_h_p, y_h_p, xy_h_p = tf_utility.from_heatmap_to_point(vec_hm_p[i], 5, 1) x_h_t, y_h_t, xy_h_t = tf_utility.from_heatmap_to_point(vec_hm_t[i], 5, 1) '''normalise points to be in [0, 1]''' x_h_p = x_h_p / 56 y_h_p = y_h_p / 56 xy_h_p = xy_h_p / 56 x_h_t = x_h_t / 56 y_h_t = y_h_t / 56 xy_h_t = xy_h_t / 56 '''test print images''' # imgpr.print_image_arr(i + 1, np.zeros(shape=[56, 56]), x_h_t, y_h_t) # imgpr.print_image_arr((i + 1)*1000, np.zeros(shape=[56, 56]), x_h_p, y_h_p) # print('--xy_h_p:---') # print(xy_h_p) # print('--xy_h_t:---') # print(xy_h_t) face_p, mouth_p, nose_p, leye_p, reye_p = self._depart_facial_point(xy_h_p) face_t, mouth_t, nose_t, leye_t, reye_t = self._depart_facial_point(xy_h_t) '''generate facial distance matrix''' face_p_mat, face_t_mat = self._generate_distance_matrix(face_p), self._generate_distance_matrix(face_t) mouth_p_mat, mouth_t_mat = self._generate_distance_matrix(mouth_p), self._generate_distance_matrix(mouth_t) nose_p_mat, nose_t_mat = self._generate_distance_matrix(nose_p), self._generate_distance_matrix(nose_t) leye_p_mat, leye_t_mat = self._generate_distance_matrix(leye_p), self._generate_distance_matrix(leye_t) reye_p_mat, reye_t_mat = self._generate_distance_matrix(reye_p), self._generate_distance_matrix(reye_t) '''calculate loss from each pair matrices''' face_loss = LearningConfig.reg_term_face * self._calculate_mse(face_p_mat, face_t_mat) / len(face_p) mouth_loss = LearningConfig.reg_term_mouth * self._calculate_mse(mouth_p_mat, mouth_t_mat) / len(mouth_p) nose_loss = LearningConfig.reg_term_nose * self._calculate_mse(nose_p_mat, nose_t_mat) / len(nose_p) leye_loss = LearningConfig.reg_term_leye * self._calculate_mse(leye_p_mat, leye_t_mat) / len(leye_p) reye_loss = LearningConfig.reg_term_reye * self._calculate_mse(reye_p_mat, reye_t_mat) / len(reye_p) loss_array.append(face_loss + mouth_loss + nose_loss + leye_loss + reye_loss) # print('mse[i]: ' + str(vec_mse[i])) # print('face_loss[i]: ' + str(face_loss)) # print('mouth_loss[i]: ' + str(mouth_loss)) # print('nose_loss[i]: ' + str(nose_loss)) # print('leye_loss[i]: ' + str(leye_loss)) # print('reye_loss[i]: ' + str(reye_loss)) # print('============') loss_array = np.array(loss_array) tensor_distance_loss = K.variable(loss_array) # tensor_total_loss = tf.reduce_mean([tensor_mean_square_error, loss_array]) tensor_total_loss = tf.add(tensor_mean_square_error, tensor_distance_loss) return tensor_total_loss
def __customLoss_base(self, yTrue, yPred): pca_utility = PCAUtility() image_utility = ImageUtility() tf_record_utility = TFRecordUtility() eigenvalues, eigenvectors, meanvector = pca_utility.load_pca_obj(DatasetName.ibug) # yTrue = tf.constant([[1.0, 2.0, 3.0], [5.0, 4.0, 7.0]]) # yPred = tf.constant([[9.0, 1.0, 2.0], [7.0, 3.0, 8.0]]) # session = K.get_session() tensor_mean_square_error = K.mean(K.square(yPred - yTrue), axis=-1) # tensor_mean_square_error = keras.losses.mean_squared_error(yPred, yTrue) mse = K.eval(tensor_mean_square_error) yPred_arr = K.eval(yPred) yTrue_arr = K.eval(yTrue) loss_array = [] for i in range(LearningConfig.batch_size): asm_loss = 0 truth_vector = yTrue_arr[i] predicted_vector = yPred_arr[i] b_vector_p = self.calculate_b_vector(predicted_vector, True, eigenvalues, eigenvectors, meanvector) y_pre_asm = meanvector + np.dot(eigenvectors, b_vector_p) """in order to test the results after PCA, you can use these lines of code""" # landmark_arr_xy, landmark_arr_x, landmark_arr_y = image_utility.create_landmarks_from_normalized(truth_vector, 224, 224, 112, 112) # image_utility.print_image_arr(i, np.ones([224, 224]), landmark_arr_x, landmark_arr_y) # # landmark_arr_xy_new, landmark_arr_x_new, landmark_arr_y_new= image_utility.create_landmarks_from_normalized(y_pre_asm, 224, 224, 112, 112) # image_utility.print_image_arr(i*100, np.ones([224, 224]), landmark_arr_x_new, landmark_arr_y_new) for j in range(len(y_pre_asm)): asm_loss += (truth_vector[j] - y_pre_asm[j]) ** 2 asm_loss /= len(y_pre_asm) # asm_loss *= mse[i] # asm_loss *= LearningConfig.regularization_term loss_array.append(asm_loss) print('mse[i]' + str(mse[i])) print('asm_loss[i]' + str(asm_loss)) print('============' ) loss_array = np.array(loss_array) tensor_asm_loss = K.variable(loss_array) # sum_loss_tensor = tf.math.add(tensor_mean_square_error, tensor_asm_loss) tensor_total_loss = tf.reduce_mean([tensor_mean_square_error, tensor_asm_loss], axis=0) # sum_loss = np.array(K.eval(tensor_asm_loss)) # print(mse) # print(K.eval(tensor_mean_square_error)) # print(K.eval(tensor_asm_loss)) # print('asm_loss ' + str(loss_array[0])) # print('mse_loss ' + str(mse[0])) # print('sum_loss ' + str(sum_loss[0])) # print('total_loss ' + str(total_loss[0])) # print(' ') return tensor_total_loss
def train(self, teachers_arch, teachers_weight_files, teachers_weight_loss, teachers_tf_train_paths, student_weight_file): """ :param teachers_arch: an array containing architecture of teacher networks :param teachers_weight_files: an array containing teachers h5 files :param teachers_weight_loss: an array containing weight of teachers model in loss function :param teachers_tf_train_paths: an array containing path of train tf records :param student_weight_file : student h5 weight path :return: null """ tf_record_util = TFRecordUtility(self.output_len) c_loss = Custom_losses() '''-------------------------------------''' ''' preparing student models ''' '''-------------------------------------''' teacher_models = [] cnn = CNNModel() for i in range(len(teachers_arch)): student_train_images, student_train_landmarks = tf_record_util.create_training_tensor_points( tfrecord_filename=teachers_tf_train_paths[i], batch_size=self.BATCH_SIZE) model = cnn.get_model(train_images=student_train_images, arch=teachers_arch[i], num_output_layers=1, output_len=self.output_len, input_tensor=None) model.load_weights(teachers_weight_files[i]) teacher_models.append(model) '''---------------------------------''' ''' creating student model ''' '''---------------------------------''' '''retrieve tf data''' train_images, train_landmarks = tf_record_util.create_training_tensor_points( tfrecord_filename=self.tf_train_path, batch_size=self.BATCH_SIZE) validation_images, validation_landmarks = tf_record_util.create_training_tensor_points( tfrecord_filename=self.tf_eval_path, batch_size=self.BATCH_SIZE) '''create model''' student_model = cnn.get_model(train_images=train_images, arch=self.arch, num_output_layers=1, output_len=self.output_len, input_tensor=train_images, inp_shape=None) if student_weight_file is not None: student_model.load_weights(student_weight_file) '''prepare callbacks''' callbacks_list = self._prepare_callback() ''' define optimizers''' optimizer = Adam(lr=1e-3, beta_1=0.9, beta_2=0.999, decay=1e-5, amsgrad=False) '''create loss''' # file = open("map_aug" + self.dataset_name, 'rb') file = open("map_orig" + self.dataset_name, 'rb') landmark_img_map = pickle.load(file) file.close() # loss_func = c_loss.custom_teacher_student_loss_cos(img_path=self.img_path, lnd_img_map=landmark_img_map, # teacher_models=teacher_models, # teachers_weight_loss=teachers_weight_loss, # bath_size=self.BATCH_SIZE, # num_points=self.output_len, cos_weight=cos_weight) loss_func = c_loss.custom_teacher_student_loss( img_path=self.img_path, lnd_img_map=landmark_img_map, teacher_models=teacher_models, teachers_weight_loss=teachers_weight_loss, bath_size=self.BATCH_SIZE, num_points=self.output_len, ds_name=self.dataset_name, loss_type=0) '''compiling model''' student_model.compile(loss=loss_func, optimizer=optimizer, metrics=['mse', 'mae'], target_tensors=train_landmarks) print('< ========== Start Training Student============= >') history = student_model.fit( train_images, train_landmarks, epochs=self.EPOCHS, steps_per_epoch=self.STEPS_PER_EPOCH, validation_data=(validation_images, validation_landmarks), validation_steps=self.STEPS_PER_VALIDATION_EPOCH, verbose=1, callbacks=callbacks_list)
from tf_record_utility import TFRecordUtility from configuration import DatasetName, DatasetType, AffectnetConf, IbugConf, W300Conf, InputDataSize from cnn_model import CNNModel from pca_utility import PCAUtility from image_utility import ImageUtility import numpy as np from train import Train from test import Test if __name__ == '__main__': tf_record_util = TFRecordUtility() pca_utility = PCAUtility() cnn_model = CNNModel() image_utility = ImageUtility() # trainer = Train(use_tf_record=False, # dataset_name=DatasetName.ibug, # custom_loss=False, # arch='mn_main', # inception_mode=True, # num_output_layers=1, # point_wise=True, # weight=None # ) # tester = Test() # trainer = Train(use_tf_record=True, # dataset_name=DatasetName.ibug, # custom_loss=False, # arch='mn_r',