def eval_pred(pred, answer_index, round_id, gt_relevance): """ Evaluate the predict results and report metrices. Only for val split. Parameters: ----------- pred: ndarray of shape (n_samples, n_rounds, n_options). answer_index: ndarray of shape (n_sample, n_rounds). round_id: ndarray of shape (n_samples, ). gt_relevance: ndarray of shape (n_samples, n_options). Returns: -------- None """ # Convert them to torch tensor to use visdialch.metrics pred = torch.Tensor(pred) answer_index = torch.Tensor(answer_index).long() round_id = torch.Tensor(round_id).long() gt_relevance = torch.Tensor(gt_relevance) sparse_metrics = SparseGTMetrics() ndcg = NDCG() sparse_metrics.observe(pred, answer_index) pred = pred[torch.arange(pred.size(0)), round_id - 1, :] ndcg.observe(pred, gt_relevance) all_metrics = {} all_metrics.update(sparse_metrics.retrieve(reset=True)) all_metrics.update(ndcg.retrieve(reset=True)) for metric_name, metric_value in all_metrics.items(): print(f"{metric_name}: {metric_value}")
temp_train_batch[key] = batch[key].to(device) elif key in ['ques', 'opt', 'ques_len', 'opt_len', 'ans_ind']: temp_train_batch[key] = batch[key][:, rnd].to(device) elif key in ['hist_len', 'hist']: temp_train_batch[key] = batch[key][:, :rnd + 1].to(device) else: pass return temp_train_batch model.eval() for i, batch in enumerate(val_dataloader): batchsize = batch['img_ids'].shape[0] rnd = 0 temp_train_batch = get_1round_batch_data(batch, rnd) output = model(temp_train_batch).view(-1, 1, 100).detach() for rnd in range(1, 10): temp_train_batch = get_1round_batch_data(batch, rnd) output = torch.cat((output, model(temp_train_batch).view(-1, 1, 100).detach()), dim=1) sparse_metrics.observe(output, batch["ans_ind"]) if "relevance" in batch: output = output[torch.arange(output.size(0)), batch["round_id"] - 1, :] ndcg.observe(output.view(-1, 100), batch["relevance"].contiguous().view(-1, 100)) # if i > 5: #for debug(like the --overfit) # break all_metrics = {} all_metrics.update(sparse_metrics.retrieve(reset=True)) all_metrics.update(ndcg.retrieve(reset=True)) for metric_name, metric_value in all_metrics.items(): print(f"{metric_name}: {metric_value}") model.train()
summary_writer.add_scalar("train/loss", batch_loss, global_iteration_step) summary_writer.add_scalar("train/lr", optimizer.param_groups[0]["lr"], global_iteration_step) if optimizer.param_groups[0]["lr"] > config["solver"]["minimum_lr"]: scheduler.step() global_iteration_step += 1 # -------------------------------------------------------------------------------------------- # ON EPOCH END (checkpointing and validation) # -------------------------------------------------------------------------------------------- checkpoint_manager.step() # validate and report automatic metrics if args.validate: print(f"\nValidation after epoch {epoch}:") for i, batch in enumerate(tqdm(val_dataloader)): for key in batch: batch[key] = batch[key].to(device) with torch.no_grad(): output = model(batch) sparse_metrics.observe(output, batch["ans_ind"]) if "gt_relevance" in batch: output = output[torch.arange(output.size(0)), batch["round_id"] - 1, :] ndcg.observe(output, batch["gt_relevance"]) all_metrics = {} all_metrics.update(sparse_metrics.retrieve(reset=True)) all_metrics.update(ndcg.retrieve(reset=True)) for metric_name, metric_value in all_metrics.items(): print(f"{metric_name}: {metric_value}") summary_writer.add_scalars("metrics", all_metrics, global_iteration_step)
class PredictionsAnalyzer: """ gt_index, gt_ans_relevance are found by get_dialog_by_row_index gt_index -> 0-index gt_round_id -> 1-index ranks -> 1-index ranks_json['round_id'] -> 1-index gt_relevance -> List # ranks_json => list of dic -> dict_keys(['image_id', 'round_id', 'ranks']) # annotations_json list of dic -> dict_keys(['image_id', 'round_id', 'gt_relevance']) # dialog_json list of dic -> dict_keys(['questions', 'answers', 'dialog']) -> [''image_id', 'answer_options'] Answer options -> [3456, 677, 888] # option idx from all answers gt_relevance -> [0.4, 0.5, ....] # relevance of options by index Ranks json -> [100, 1, ....] # ranks of all options Indices are same as before..but their new ranks are shown instead of their indices being shown Eg. [5, 6, 8, ...] means that original 0-index option is now 5th rank """ def __init__(self, path_val_data, dense_annotations_jsonpath, path_images_root, model_preds_root): self.path_val_data = path_val_data self.dense_annotations_jsonpath = dense_annotations_jsonpath # Ideally return the q and a here self.read_data() self.path_images_root = path_images_root self.model_preds_root = model_preds_root self.img_folder_list = self.get_img_folder_list(self.path_images_root) self.img_map = self.get_img_map(self.img_folder_list) self.models_list = self.get_model_type_list(self.model_preds_root) self.gt_indices_list = [] # 0-indexed self.gt_relevance_list = [] self.ndcg = NDCG(is_direct_ranks=True) def get_models_list(self): return self.models_list def read_data(self): self.data_val = json.load(open(self.path_val_data)) self.questions = self.data_val['data']['questions'] print("Total questions:", len(self.questions)) self.answers = self.data_val['data']['answers'] print("Total answers:", len(self.answers)) self.annotations_json = json.load(open(self.dense_annotations_jsonpath)) self.dialogs = self.data_val['data']['dialogs'] print(f"Length of all dialogs: {len(self.dialogs)}") print(f"Length of all annotations: {len(self.annotations_json)}") return @staticmethod def get_img_folder_list(path_images_root, image_folder_name="VisualDialog_val2018"): path_visdial_val = os.path.join(path_images_root, image_folder_name) img_folder_list = glob.glob(os.path.join(path_visdial_val, '*')) print("Total images in folder:", len(img_folder_list)) return img_folder_list @staticmethod def get_model_type_list(model_preds_root): model_folder_list = [os.path.basename(x) for x in glob.glob(os.path.join(model_preds_root, '*'))] print("Total models in folder:", len(model_folder_list)) return model_folder_list @staticmethod def json_load(file_path): with open(file_path, "r") as fb: data = json.load(fb) return data @staticmethod def convert_list_json_dic(ranks_json): image_ranks_dic = {} for i in range(len(ranks_json)): image_ranks_dic[ranks_json[i]["image_id"]] = ranks_json[i] return image_ranks_dic @staticmethod def image_id_from_path(image_path): """Given a path to an image, return its id. Parameters ---------- image_path : str Path to image, e.g.: coco_train2014/COCO_train2014/000000123456.jpg img_name = "VisualDialog_val2018_000000254080.jpg" Returns ------- int Corresponding image id (123456) """ return int(image_path.split("/")[-1][-16:-4]) def get_img_map(self, img_folder_list): img_map = dict() for img_path in img_folder_list: img_id = self.image_id_from_path(img_path) img_map[img_id] = img_path return img_map def show_img(self, img_id): img_path = self.img_map[img_id] print("Reading image from: ", img_path) plt.imshow(plt.imread(img_path)) def get_both_phase_ranks(self, model_type, ranks_phase1_file="ranks_val_12_crowdsourced.json", ranks_finetune_file="ranks_val_best_ndcg_crowdsourced.json"): """ :param model_type: :param ranks_phase1_file: :param ranks_finetune_file: :return: """ model_rank_phase1_path = Path(self.model_preds_root, model_type, ranks_phase1_file) model_rank_finetune_path = Path(self.model_preds_root, model_type, ranks_finetune_file) # list of dic -> dict_keys(['image_id', 'round_id', 'ranks']) ranks_phase1_json = self.json_load(model_rank_phase1_path) ranks_finetune_json = self.json_load(model_rank_finetune_path) return ranks_phase1_json, ranks_finetune_json def subset_val_ranks_with_dense_annotation(self, ranks_json, top_k=5): """ this is because val ranks consists of 10 turns :param ranks_json: :param top_k: :return: """ rank_dense_list = [] relevance_dic = {} index_dic = {} gt_results_index_dic = {} # gt_results_relevance_dic = {} gt_indices_list = [] # 0-indexed gt_relevance_list = [] dialogs = self.data_val['data']['dialogs'] for i in range(len(self.annotations_json)): # They will be in same order by image_id round_id = self.annotations_json[i]['round_id'] - 1 # 0-indexing index_for_ranks_json = i * 10 + round_id # for each image: 10 assert ranks_json[index_for_ranks_json]['round_id'] == round_id + 1 # Check with 1-indexing assert ranks_json[index_for_ranks_json]['image_id'] == self.annotations_json[i]['image_id'] gt_relevance = self.annotations_json[i]['gt_relevance'] rank_dense_list.append(ranks_json[index_for_ranks_json]) ranks = ranks_json[index_for_ranks_json]['ranks'] relevance_sum = 0 image_id = ranks_json[index_for_ranks_json]['image_id'] # To actually have the indices_list and relevance list before hand assert image_id == dialogs[i]['image_id'] gt_index = dialogs[i]['dialog'][round_id]['gt_index'] # round_id already 0-index gt_index also 0-indexed gt_ans_relevance = gt_relevance[gt_index] gt_indices_list.append(gt_index) gt_relevance_list.append(gt_ans_relevance) # We need to find rank of (gt_index + 1) - coz ranks are 1-indexed pred_index = ranks.index(gt_index + 1) # maintaining 0-index if pred_index in gt_results_index_dic: # gt_results_index_dic[pred_index].append(image_id) gt_results_index_dic[pred_index].append(i) else: # gt_results_index_dic[pred_index] = [image_id] gt_results_index_dic[pred_index] = [i] for j in range(top_k): relevance_sum += gt_relevance[ranks[j] - 1] # 0-indexing # We keep a list of relevance sum - kind of proxy to get the best scores if relevance_sum in relevance_dic: relevance_dic[relevance_sum].append(image_id) index_dic[relevance_sum].append(i) else: relevance_dic[relevance_sum] = [image_id] index_dic[relevance_sum] = [i] # Re-assign self.gt_indices_list = gt_indices_list self.gt_relevance_list = gt_relevance_list return rank_dense_list, relevance_dic, index_dic, gt_results_index_dic def print_dialog(self, dialog, gt_round_id, gt_ans_relevance, answer_options): """ dialog -> per image gt_round_id -> should be 1-index """ print("\n") print("Dialog: ") print("\n") for round_indx in range(gt_round_id - 1): print("Q{}".format(round_indx + 1), f"{self.questions[dialog[round_indx]['question']].capitalize()}?") print("A{}".format(round_indx + 1), f"{self.answers[dialog[round_indx]['answer']].capitalize()}.") print("\n") print("Question {}: ".format(gt_round_id), f"{self.questions[dialog[gt_round_id - 1]['question']].capitalize()}?") print("\n") print("GT answer: ", f"{self.answers[dialog[gt_round_id - 1]['answer']].capitalize()}.") gt_ans_index = dialog[gt_round_id - 1]['answer'] found_ans_index = answer_options.index(gt_ans_index) # returns 0-index # print("GT index: ", dialog[gt_round_id - 1]['gt_index']) gt_index = dialog[gt_round_id - 1]['gt_index'] assert found_ans_index == gt_index print("GT relevance: ", gt_ans_relevance) print("\n") def get_dialog_by_row_index(self, row_index, is_print): """ :param row_index: defines the whole dialog (not turn) :return: """ caption = self.data_val['data']['dialogs'][row_index]['caption'] dialog = self.data_val['data']['dialogs'][row_index]['dialog'] image_id = self.data_val['data']['dialogs'][row_index]['image_id'] # This is for turn dense_annotations = self.annotations_json[row_index] # print(dense_annotations.keys()) gt_round_id = dense_annotations["round_id"] # 1-index gt_image_id = dense_annotations["image_id"] assert gt_image_id == image_id gt_relevance = dense_annotations["gt_relevance"] assert len(gt_relevance) == 100 gt_index = dialog[gt_round_id - 1]['gt_index'] gt_ans_relevance = gt_relevance[gt_index] answer_options = dialog[gt_round_id - 1]["answer_options"] if is_print: print(caption) # self.show_img(image_id) self.print_dialog(dialog, gt_round_id, gt_ans_relevance, answer_options) non_zero_relevant_ans = np.count_nonzero(gt_relevance) print("Number of answers with non-zero relevance: ", non_zero_relevant_ans) return answer_options, gt_index, gt_relevance, gt_round_id def print_top_k_preds(self, ranks_model, gt_index, gt_relevance, answer_options, phase, top_k=5): """ gt_index -> should be 0-index """ print("\n") print(f"{phase} Phase: ") # Get rank of gt_index # pred_index = ranks_model.index(gt_index) + 1 # to convert to 1-index # print("GT answer predicted at: ", pred_index) # print("GT answer is: ", self.answers[answer_options[ranks_model[pred_index-1]]]) # reverse_ranks = ranks_model[::-1] # print(reverse_ranks) # print(ranks_model) # print(self.get_max_index(gt_relevance)) print("GT predicted rank: ", ranks_model[gt_index] +1) # gt rank and ranks both 0-index # indices are same. but their ranks are shown now. instead of option number being shown. indices_list = sorted(range(len(ranks_model)), key=lambda i: ranks_model[i])[:top_k] self.get_ndcg_value_wrapper(ranks=ranks_model, gt_relevance=gt_relevance) for i in range(top_k): print("Relevance:", gt_relevance[indices_list[i]], "Answer:", f"{self.answers[answer_options[indices_list[i]]].capitalize()}.") print("\n") def get_ndcg_value_wrapper(self, ranks: List, gt_relevance : List): """ :param ranks: list :param gt_relevance: list :return: """ # (batch_size, num_options) ranks = torch.tensor(ranks).float() gt_relevance = torch.tensor(gt_relevance).float() # If individual sample, we need to add 0-dim if len(ranks.size()) == 1: ranks = ranks.unsqueeze(0) gt_relevance = gt_relevance.unsqueeze(0) self.ndcg.observe(ranks, gt_relevance) value = self.ndcg.retrieve(reset=True)["ndcg"] print(f"NDCG: {round(value*100,2)}") return value def print_top_k_preds_wrapper(self, ranks_phase1_json, ranks_finetune_json, gt_index, gt_relevance, answer_options, row_index, gt_round_id, top_k=5): ranks_phase1 = ranks_phase1_json[row_index]["ranks"] ranks_finetune = ranks_finetune_json[row_index]["ranks"] assert ranks_phase1_json[row_index]["round_id"] == gt_round_id # Ranks are 1 indexed - shifting to 0 ranks_phase1 = [rank - 1 for rank in ranks_phase1] ranks_finetune = [rank - 1 for rank in ranks_finetune] self.print_top_k_preds(ranks_phase1, gt_index, gt_relevance, answer_options, "Spare Annotation", top_k=top_k) self.print_top_k_preds(ranks_finetune, gt_index, gt_relevance, answer_options, "Curriculum Finetuning", top_k=top_k) # Main api open to call def get_analysis(self, model_type, top_k=5, row_index=1, is_print=False): ranks_phase1_json, ranks_finetune_json = self.get_both_phase_ranks(model_type) # print(f"Length of ranks phase 1 json was earlier: {len(ranks_phase1_json)}") # print(f"Length of ranks finetune json was earlier: {len(ranks_finetune_json)}") ranks_phase1_json, relevance_dic_phase1, index_dic_phase1,\ gt_results_index_dic_phase1 = self.subset_val_ranks_with_dense_annotation( ranks_phase1_json, top_k=top_k) # print(f"After subset Length of ranks phase 1 json : {len(ranks_phase1_json)}") ranks_finetune_json, relevance_dic_finetune, \ index_dic_finetune, gt_results_index_dic_finetune = self.subset_val_ranks_with_dense_annotation( ranks_finetune_json, top_k=top_k) # print(f"After subset Length of ranks finetune json : {len(ranks_phase1_json)}") # For identifying row_index # print(relevance_dic_phase1) # print(relevance_dic_finetune) # print(index_dic_finetune) answer_options, gt_index, gt_relevance, gt_round_id = self.get_dialog_by_row_index(row_index, is_print) self.print_top_k_preds_wrapper(ranks_phase1_json, ranks_finetune_json, gt_index, gt_relevance, answer_options, row_index, gt_round_id, top_k=top_k) # return relevance_dic_phase1, relevance_dic_finetune, index_dic_finetune return # Return only once to find subsets def get_dic_models(self, model_type, top_k=5): ranks_phase1_json, ranks_finetune_json = self.get_both_phase_ranks(model_type) ranks_phase1_json, relevance_dic_phase1, \ index_dic_phase1, gt_results_index_dic_phase1\ = self.subset_val_ranks_with_dense_annotation( ranks_phase1_json, top_k=top_k) ranks_finetune_json, relevance_dic_finetune, \ index_dic_finetune, gt_results_index_dic_finetune = self.subset_val_ranks_with_dense_annotation( ranks_finetune_json, top_k=top_k) return relevance_dic_phase1, relevance_dic_finetune, index_dic_phase1, \ index_dic_finetune, gt_results_index_dic_phase1, \ gt_results_index_dic_finetune @staticmethod def get_max_index(values): # Returns 0-index _index = values.index(max(values)) return _index def list_ans_opts(self, answer_options, gt_relevance, gt_index, num_ans_opts=5): """ To list answer options for the task :param answer_options: :param num_ans_opts: :return: """ print_ans_opt_list = [] print_relevance = [] for ans_opt in range(num_ans_opts-1): print_ans_opt_list.append(self.answers[answer_options[ans_opt]]) print_relevance.append(gt_relevance[ans_opt]) # print(self.answers[answer_options[ans_opt]]) # print(gt_relevance[ans_opt]) # One we will print for max index max_score_index = self.get_max_index(gt_relevance) # 0-index print_ans_opt_list.append(self.answers[answer_options[max_score_index]]) print_relevance.append(gt_relevance[max_score_index]) print_ans_opt_list.append(self.answers[answer_options[gt_index]]) print_relevance.append(gt_relevance[gt_index]) print("Answers:") self.print_line_by_line(print_ans_opt_list) print("Relevance:") self.print_line_by_line(print_relevance) print("Last one is gt") # print(print_ans_opt_list) # print(print_relevance) # print(self.answers[answer_options[max_score_index]]) # print(gt_relevance[max_score_index]) @staticmethod def print_line_by_line(mylist): for elem in mylist: print(elem)
class SubsetComplementVisDial: """ We will be subsetting actual dataset based on subset image ids """ def __init__(self, config): super().__init__() self.ndcg = NDCG(is_direct_ranks=True) # We are calculating NDCG directly based on ranks # self.path_val_data = config.path_val_data self.dense_annotations_jsonpath = config.dense_annotations_jsonpath self.model_preds_root = config.model_preds_root self.models_list = self.get_model_type_list(self.model_preds_root) self.annotations_json = json.load(open( self.dense_annotations_jsonpath)) self.hist_info_images = [ 257366, 425477, 191097, 552399, 12468, 458949, 109735, 311793, 437200, 355853, 98849, 57743, 83289, 488471, 446567, 196905, 308846, 328336, 289233, 52156, 366462, 511748, 457675, 518811, 413085, 432039, 531270, 430580, 293582, 544148, 80366, 179366, 150236, 400960, 10424, 451398, 498340, 268914, 384171, 172461, 387266, 214227, 555578, 181772, 149373, 251385, 407878, 574545, 544827, 120559, 19299, 73638, 496822, 204195, 97073, 209447, 53433, 403234, 524006, 178300, 376460, 570468, 292100, 227006, 170315, 456824, 525726, 179064, 98879, 558975, 193521, 377823, 449230, 44468, 573552, 288308, 237956, 69538, 250654, 439842, 146314, 458818, 122826, 33976, 322815, 239030, 209271, 560666, 361734, 225491, 27366, 29060, 191186, 394073, 120870, 580183, 111013 ] self.subset_type = "_complement" def extract_ndcg(self, model_type: str, num_samples: int = 97): """ :param model_type: for which we want to extract :return: """ try: ranks_phase1_file = f"ranks_val_12.json" ranks_finetune_file = f"ranks_val_best_ndcg.json" ranks_phase1_json, ranks_finetune_json = self.get_both_phase_ranks( model_type, ranks_phase1_file=ranks_phase1_file, ranks_finetune_file=ranks_finetune_file) except: print( f"For Model {model_type}, we are using 11 as the ckpt based on ndcg" ) ranks_phase1_file = f"ranks_val_11.json" ranks_finetune_file = f"ranks_val_best_ndcg.json" ranks_phase1_json, ranks_finetune_json = self.get_both_phase_ranks( model_type, ranks_phase1_file=ranks_phase1_file, ranks_finetune_file=ranks_finetune_file) # ranks_phase1_json, ranks_finetune_json = self.get_both_phase_ranks(model_type) # Remove all examples considered as hist info ranks_phase1_json, new_annotation_list = self.subset_val_ranks_with_dense_annotation( ranks_phase1_json) ranks_finetune_json, new_annotation_list = self.subset_val_ranks_with_dense_annotation( ranks_finetune_json) # Sample all elements here indices_list = random.sample(range(len(ranks_phase1_json)), k=num_samples) ranks_phase1_json = [ranks_phase1_json[i] for i in indices_list] ranks_finetune_json = [ranks_finetune_json[i] for i in indices_list] new_annotation_list = [new_annotation_list[i] for i in indices_list] assert len(ranks_phase1_json) == len(ranks_finetune_json) == len( new_annotation_list) gt_relevance_list = [] ranks_list_phase1 = [] ranks_list_finetune = [] ndcg_list_phase1 = [] ndcg_list_finetune = [] for indx in range(len(ranks_phase1_json)): ranks_phase1 = ranks_phase1_json[indx]["ranks"] ranks_finetune = ranks_finetune_json[indx]["ranks"] gt_relevance = new_annotation_list[indx]['gt_relevance'] # Assert if we are doing for same round ids assert ranks_phase1_json[indx]['round_id'] == new_annotation_list[ indx]['round_id'] assert ranks_finetune_json[indx][ 'round_id'] == new_annotation_list[indx]['round_id'] ndcg_sample_phase1 = self.get_ndcg_value_wrapper( ranks_phase1, gt_relevance) ndcg_sample_finetune = self.get_ndcg_value_wrapper( ranks_finetune, gt_relevance) # Maintain the list for individual samples ndcg_list_phase1.append(ndcg_sample_phase1) ndcg_list_finetune.append(ndcg_sample_finetune) # For whole set - for verification ranks_list_phase1.append(ranks_phase1) ranks_list_finetune.append(ranks_finetune) gt_relevance_list.append(gt_relevance) # For whole val set, for verification..Not saving them! # ndcg_sample_phase1 = self.get_ndcg_value_wrapper(ranks_list_phase1, gt_relevance_list) # ndcg_sample_finetune = self.get_ndcg_value_wrapper(ranks_list_finetune, gt_relevance_list) # print(f"NDCG for the whole set for {model_type} (phase1): ", ndcg_sample_phase1*100) # print(f"NDCG for the whole set for {model_type}(finetune)", ndcg_sample_finetune*100) ndcg_write_path = self._get_ndcg_path(self.model_preds_root, model_type, phase="sparse", subset_type=self.subset_type) print(f"Saving as {ndcg_write_path}") self.write_list_to_file(ndcg_write_path, ndcg_list_phase1) ndcg_write_path = self._get_ndcg_path(self.model_preds_root, model_type, phase="finetune", subset_type=self.subset_type) print(f"Saving as {ndcg_write_path}") self.write_list_to_file(ndcg_write_path, ndcg_list_finetune) def get_ndcg_value_wrapper(self, ranks: List, gt_relevance: List): """ :param ranks: list :param gt_relevance: list :return: """ # (batch_size, num_options) ranks = torch.tensor(ranks).float() gt_relevance = torch.tensor(gt_relevance).float() # If individual sample, we need to add 0-dim if len(ranks.size()) == 1: ranks = ranks.unsqueeze(0) gt_relevance = gt_relevance.unsqueeze(0) self.ndcg.observe(ranks, gt_relevance) value = self.ndcg.retrieve(reset=True)["ndcg"] return value # SA: TODO - Notice ranks file changed here def get_both_phase_ranks(self, model_type, ranks_phase1_file="ranks_val_12.json", ranks_finetune_file="ranks_val_best_ndcg.json"): """ :param model_type: :param ranks_phase1_file: :param ranks_finetune_file: :return: """ model_rank_phase1_path = Path(self.model_preds_root, model_type, ranks_phase1_file) model_rank_finetune_path = Path(self.model_preds_root, model_type, ranks_finetune_file) # list of dic -> dict_keys(['image_id', 'round_id', 'ranks']) ranks_phase1_json = self.json_load(model_rank_phase1_path) ranks_finetune_json = self.json_load(model_rank_finetune_path) return ranks_phase1_json, ranks_finetune_json def subset_val_ranks_with_dense_annotation(self, ranks_json): """ this is because val ranks consists of 10 turns :param ranks_json: :param top_k: :return: """ rank_dense_list = [] new_annotation_list = [] for i in range(len(self.annotations_json)): # They will be in same order by image_id round_id = self.annotations_json[i]['round_id'] - 1 # 0-indexing index_for_ranks_json = i * 10 + round_id # for each image: 10 assert ranks_json[index_for_ranks_json][ 'round_id'] == round_id + 1 # Check with 1-indexing assert ranks_json[index_for_ranks_json][ 'image_id'] == self.annotations_json[i]['image_id'] # Subset the data here image_id = self.annotations_json[i]['image_id'] if image_id not in self.hist_info_images: rank_dense_list.append(ranks_json[index_for_ranks_json]) new_annotation_list.append(self.annotations_json[i]) return rank_dense_list, new_annotation_list @staticmethod def write_list_to_file(filepath, write_list) -> None: with open(filepath, 'w') as file_handler: for item in write_list: file_handler.write("{}\n".format(item)) # outfile.write("\n".join(itemlist)) return @staticmethod def json_load(file_path): with open(file_path, "r") as fb: data = json.load(fb) return data @staticmethod def get_model_type_list(model_preds_root) -> List: model_folder_list = [ os.path.basename(x) for x in glob.glob(os.path.join(model_preds_root, '*')) ] print("Total models in folder:", len(model_folder_list)) return model_folder_list @staticmethod def _get_ndcg_path(model_root: str, model_type: str, phase: str, subset_type: str = '_complement', ext: str = 'txt') -> str: json_path = f"{model_root}/{model_type}/ndcg_{phase}_{model_type}{subset_type}.{ext}" return json_path
def train(config, args, dataloader_dic, device, finetune: bool = False, load_pthpath: str = "", finetune_regression: bool = False, dense_scratch_train: bool = False, dense_annotation_type: str = "default"): """ :param config: :param args: :param dataloader_dic: :param device: :param finetune: :param load_pthpath: :param finetune_regression: :param dense_scratch_train: when we want to start training only on 2000 annotations :param dense_annotation_type: default :return: """ # ============================================================================= # SETUP BEFORE TRAINING LOOP # ============================================================================= train_dataset = dataloader_dic["train_dataset"] train_dataloader = dataloader_dic["train_dataloader"] val_dataloader = dataloader_dic["val_dataloader"] val_dataset = dataloader_dic["val_dataset"] model = get_model(config, args, train_dataset, device) if finetune and not dense_scratch_train: assert load_pthpath != "", "Please provide a path" \ " for pre-trained model before " \ "starting fine tuning" print(f"\n Begin Finetuning:") optimizer, scheduler, iterations, lr_scheduler_type = get_solver( config, args, train_dataset, val_dataset, model, finetune=finetune) start_time = datetime.datetime.strftime(datetime.datetime.utcnow(), '%d-%b-%Y-%H:%M:%S') if args.save_dirpath == 'checkpoints/': args.save_dirpath += '%s+%s/%s' % ( config["model"]["encoder"], config["model"]["decoder"], start_time) summary_writer = SummaryWriter(log_dir=args.save_dirpath) checkpoint_manager = CheckpointManager(model, optimizer, args.save_dirpath, config=config) sparse_metrics = SparseGTMetrics() ndcg = NDCG() best_val_loss = np.inf # SA: initially loss can be any number best_val_ndcg = 0.0 # If loading from checkpoint, adjust start epoch and load parameters. # SA: 1. if finetuning -> load from saved model # 2. train -> default load_pthpath = "" # 3. else load pthpath if (not finetune and load_pthpath == "") or dense_scratch_train: start_epoch = 1 else: # "path/to/checkpoint_xx.pth" -> xx ### To cater model finetuning from models with "best_ndcg" checkpoint try: start_epoch = int(load_pthpath.split("_")[-1][:-4]) + 1 except: start_epoch = 1 model_state_dict, optimizer_state_dict = load_checkpoint(load_pthpath) # SA: updating last epoch checkpoint_manager.update_last_epoch(start_epoch) if isinstance(model, nn.DataParallel): model.module.load_state_dict(model_state_dict) else: model.load_state_dict(model_state_dict) # SA: for finetuning optimizer should start from its learning rate if not finetune: optimizer.load_state_dict(optimizer_state_dict) else: print("Optimizer not loaded. Different optimizer for finetuning.") print("Loaded model from {}".format(load_pthpath)) # ============================================================================= # TRAINING LOOP # ============================================================================= # Forever increasing counter to keep track of iterations (for tensorboard log). global_iteration_step = (start_epoch - 1) * iterations running_loss = 0.0 # New train_begin = datetime.datetime.utcnow() # New if finetune: end_epoch = start_epoch + config["solver"]["num_epochs_curriculum"] - 1 if finetune_regression: # criterion = nn.MSELoss(reduction='mean') # criterion = nn.KLDivLoss(reduction='mean') criterion = nn.MultiLabelSoftMarginLoss() else: end_epoch = config["solver"]["num_epochs"] # SA: normal training criterion = get_loss_criterion(config, train_dataset) # SA: end_epoch + 1 => for loop also doing last epoch for epoch in range(start_epoch, end_epoch + 1): # ------------------------------------------------------------------------- # ON EPOCH START (combine dataloaders if training on train + val) # ------------------------------------------------------------------------- if config["solver"]["training_splits"] == "trainval": combined_dataloader = itertools.chain(train_dataloader, val_dataloader) else: combined_dataloader = itertools.chain(train_dataloader) print(f"\nTraining for epoch {epoch}:") for i, batch in enumerate(tqdm(combined_dataloader)): for key in batch: batch[key] = batch[key].to(device) optimizer.zero_grad() output = model(batch) if finetune: target = batch["gt_relevance"] # Same as for ndcg validation, only one round is present output = output[torch.arange(output.size(0)), batch["round_id"] - 1, :] # SA: todo regression loss if finetune_regression: batch_loss = mse_loss(output, target, criterion) else: batch_loss = compute_ndcg_type_loss(output, target) else: batch_loss = get_batch_criterion_loss_value( config, batch, criterion, output) batch_loss.backward() optimizer.step() # -------------------------------------------------------------------- # update running loss and decay learning rates # -------------------------------------------------------------------- if running_loss > 0.0: running_loss = 0.95 * running_loss + 0.05 * batch_loss.item() else: running_loss = batch_loss.item() # SA: lambda_lr was configured to reduce lr after milestone epochs if lr_scheduler_type == "lambda_lr": scheduler.step(global_iteration_step) global_iteration_step += 1 if global_iteration_step % 100 == 0: # print current time, running average, learning rate, iteration, epoch print( "[{}][Epoch: {:3d}][Iter: {:6d}][Loss: {:6f}][lr: {:8f}]". format(datetime.datetime.utcnow() - train_begin, epoch, global_iteration_step, running_loss, optimizer.param_groups[0]['lr'])) # tensorboardX summary_writer.add_scalar("train/loss", batch_loss, global_iteration_step) summary_writer.add_scalar("train/lr", optimizer.param_groups[0]["lr"], global_iteration_step) torch.cuda.empty_cache() # ------------------------------------------------------------------------- # ON EPOCH END (checkpointing and validation) # ------------------------------------------------------------------------- if not finetune: checkpoint_manager.step(epoch=epoch) else: print("Validating before checkpointing.") # SA: ideally another function: too much work # Validate and report automatic metrics. if args.validate: # Switch dropout, batchnorm etc to the correct mode. model.eval() val_loss = 0 print(f"\nValidation after epoch {epoch}:") for i, batch in enumerate(tqdm(val_dataloader)): for key in batch: batch[key] = batch[key].to(device) with torch.no_grad(): output = model(batch) if finetune: target = batch["gt_relevance"] # Same as for ndcg validation, only one round is present out_ndcg = output[torch.arange(output.size(0)), batch["round_id"] - 1, :] # SA: todo regression loss if finetune_regression: batch_loss = mse_loss(out_ndcg, target, criterion) else: batch_loss = compute_ndcg_type_loss( out_ndcg, target) else: batch_loss = get_batch_criterion_loss_value( config, batch, criterion, output) val_loss += batch_loss.item() sparse_metrics.observe(output, batch["ans_ind"]) if "gt_relevance" in batch: output = output[torch.arange(output.size(0)), batch["round_id"] - 1, :] ndcg.observe(output, batch["gt_relevance"]) all_metrics = {} all_metrics.update(sparse_metrics.retrieve(reset=True)) all_metrics.update(ndcg.retrieve(reset=True)) for metric_name, metric_value in all_metrics.items(): print(f"{metric_name}: {metric_value}") summary_writer.add_scalars("metrics", all_metrics, global_iteration_step) model.train() torch.cuda.empty_cache() val_loss = val_loss / len(val_dataloader) print(f"Validation loss for {epoch} epoch is {val_loss}") print(f"Validation loss for batch is {batch_loss}") summary_writer.add_scalar("val/loss", batch_loss, global_iteration_step) if val_loss < best_val_loss: print(f" Best model found at {epoch} epoch! Saving now.") best_val_loss = val_loss if dense_annotation_type == "default": checkpoint_manager.save_best() else: print(f" Not saving the model at {epoch} epoch!") # SA: Saving the best model both for loss and ndcg now val_ndcg = all_metrics["ndcg"] if val_ndcg > best_val_ndcg: print(f" Best ndcg model found at {epoch} epoch! Saving now.") best_val_ndcg = val_ndcg if dense_annotation_type == "default": checkpoint_manager.save_best(ckpt_name="best_ndcg") else: # SA: trying for dense annotations ckpt_name = f"best_ndcg_annotation_{dense_annotation_type}" checkpoint_manager.save_best(ckpt_name=ckpt_name) else: print(f" Not saving the model at {epoch} epoch!") # SA: "reduce_lr_on_plateau" works only with validate for now if lr_scheduler_type == "reduce_lr_on_plateau": # scheduler.step(val_loss) # SA: # Loss should decrease while ndcg should increase! # can also change the mode in LR reduce on plateau to max scheduler.step(-1 * val_ndcg)
class NDCGForRanks: def __init__(self, config): super().__init__() self.ndcg = NDCG(is_direct_ranks=True) # We are calculating NDCG directly based on ranks # self.path_val_data = config.path_val_data self.dense_annotations_jsonpath = config.dense_annotations_jsonpath self.model_preds_root = config.model_preds_root self.models_list = self.get_model_type_list(self.model_preds_root) self.annotations_json = json.load(open( self.dense_annotations_jsonpath)) self.subset_type = config.subset_type def extract_ndcg(self, model_type: str): """ :param model_type: for which we want to extract :return: """ try: ranks_phase1_file = f"ranks_val_12{self.subset_type}.json" ranks_finetune_file = f"ranks_val_best_ndcg{self.subset_type}.json" ranks_phase1_json, ranks_finetune_json = self.get_both_phase_ranks( model_type, ranks_phase1_file=ranks_phase1_file, ranks_finetune_file=ranks_finetune_file) except: print( f"For Model {model_type}, we are using 11 as the ckpt based on ndcg" ) ranks_phase1_file = f"ranks_val_11{self.subset_type}.json" ranks_finetune_file = f"ranks_val_best_ndcg{self.subset_type}.json" ranks_phase1_json, ranks_finetune_json = self.get_both_phase_ranks( model_type, ranks_phase1_file=ranks_phase1_file, ranks_finetune_file=ranks_finetune_file) ranks_phase1_json = self.subset_val_ranks_with_dense_annotation( ranks_phase1_json) ranks_finetune_json = self.subset_val_ranks_with_dense_annotation( ranks_finetune_json) assert len(ranks_phase1_json) == len(ranks_finetune_json) == len( self.annotations_json) gt_relevance_list = [] ranks_list_phase1 = [] ranks_list_finetune = [] ndcg_list_phase1 = [] ndcg_list_finetune = [] for indx in range(len(ranks_phase1_json)): ranks_phase1 = ranks_phase1_json[indx]["ranks"] ranks_finetune = ranks_finetune_json[indx]["ranks"] gt_relevance = self.annotations_json[indx]['gt_relevance'] # Assert if we are doing for same round ids assert ranks_phase1_json[indx][ 'round_id'] == self.annotations_json[indx]['round_id'] assert ranks_finetune_json[indx][ 'round_id'] == self.annotations_json[indx]['round_id'] ndcg_sample_phase1 = self.get_ndcg_value_wrapper( ranks_phase1, gt_relevance) ndcg_sample_finetune = self.get_ndcg_value_wrapper( ranks_finetune, gt_relevance) # Maintain the list for individual samples ndcg_list_phase1.append(ndcg_sample_phase1) ndcg_list_finetune.append(ndcg_sample_finetune) # For whole set - for verification ranks_list_phase1.append(ranks_phase1) ranks_list_finetune.append(ranks_finetune) gt_relevance_list.append(gt_relevance) # For whole val set, for verification..Not saving them! ndcg_sample_phase1 = self.get_ndcg_value_wrapper( ranks_list_phase1, gt_relevance_list) ndcg_sample_finetune = self.get_ndcg_value_wrapper( ranks_list_finetune, gt_relevance_list) print(f"NDCG for the whole set for {model_type} (phase1): ", round(ndcg_sample_phase1 * 100, 2)) print(f"NDCG for the whole set for {model_type}(finetune)", round(ndcg_sample_finetune * 100, 2)) ndcg_write_path = self._get_ndcg_path(self.model_preds_root, model_type, phase="sparse", subset_type=self.subset_type) print(f"Saving as {ndcg_write_path}") self.write_list_to_file(ndcg_write_path, ndcg_list_phase1) ndcg_write_path = self._get_ndcg_path(self.model_preds_root, model_type, phase="finetune", subset_type=self.subset_type) print(f"Saving as {ndcg_write_path}") self.write_list_to_file(ndcg_write_path, ndcg_list_finetune) def get_ndcg_value_wrapper(self, ranks: List, gt_relevance: List): """ :param ranks: list :param gt_relevance: list :return: """ # (batch_size, num_options) ranks = torch.tensor(ranks).float() gt_relevance = torch.tensor(gt_relevance).float() # If individual sample, we need to add 0-dim if len(ranks.size()) == 1: ranks = ranks.unsqueeze(0) gt_relevance = gt_relevance.unsqueeze(0) self.ndcg.observe(ranks, gt_relevance) value = self.ndcg.retrieve(reset=True)["ndcg"] return value def get_both_phase_ranks( self, model_type, ranks_phase1_file="ranks_val_12_crowdsourced.json", ranks_finetune_file="ranks_val_best_ndcg_crowdsourced.json"): """ :param model_type: :param ranks_phase1_file: :param ranks_finetune_file: :return: """ model_rank_phase1_path = Path(self.model_preds_root, model_type, ranks_phase1_file) model_rank_finetune_path = Path(self.model_preds_root, model_type, ranks_finetune_file) # list of dic -> dict_keys(['image_id', 'round_id', 'ranks']) ranks_phase1_json = self.json_load(model_rank_phase1_path) ranks_finetune_json = self.json_load(model_rank_finetune_path) return ranks_phase1_json, ranks_finetune_json def subset_val_ranks_with_dense_annotation(self, ranks_json): """ this is because val ranks consists of 10 turns :param ranks_json: :param top_k: :return: """ rank_dense_list = [] # gt_indices_list = [] # 0-indexed # gt_relevance_list = [] # dialogs = self.data_val['data']['dialogs'] for i in range(len(self.annotations_json)): # They will be in same order by image_id round_id = self.annotations_json[i]['round_id'] - 1 # 0-indexing index_for_ranks_json = i * 10 + round_id # for each image: 10 assert ranks_json[index_for_ranks_json][ 'round_id'] == round_id + 1 # Check with 1-indexing assert ranks_json[index_for_ranks_json][ 'image_id'] == self.annotations_json[i]['image_id'] rank_dense_list.append(ranks_json[index_for_ranks_json]) # ranks = ranks_json[index_for_ranks_json]['ranks'] # image_id = ranks_json[index_for_ranks_json]['image_id'] # gt_relevance = self.annotations_json[i]['gt_relevance'] # # To actually have the indices_list and relevance list before hand # assert image_id == dialogs[i]['image_id'] # gt_index = dialogs[i]['dialog'][round_id]['gt_index'] # # round_id already 0-index gt_index also 0-indexed # gt_ans_relevance = gt_relevance[gt_index] # gt_indices_list.append(gt_index) # gt_relevance_list.append(gt_ans_relevance) # # We need to find rank of (gt_index + 1) - coz ranks are 1-indexed # pred_index = ranks.index(gt_index + 1) # maintaining 0-index return rank_dense_list @staticmethod def write_list_to_file(filepath, write_list) -> None: with open(filepath, 'w') as file_handler: for item in write_list: file_handler.write("{}\n".format(item)) # outfile.write("\n".join(itemlist)) return @staticmethod def json_load(file_path): with open(file_path, "r") as fb: data = json.load(fb) return data @staticmethod def get_model_type_list(model_preds_root) -> List: model_folder_list = [ os.path.basename(x) for x in glob.glob(os.path.join(model_preds_root, '*')) ] print("Total models in folder:", len(model_folder_list)) return model_folder_list @staticmethod def _get_ndcg_path(model_root: str, model_type: str, phase: str, subset_type: str = '_crowdsourced', ext: str = 'txt') -> str: json_path = f"{model_root}/{model_type}/ndcg_{phase}_{model_type}{subset_type}.{ext}" return json_path