def eval_pred(pred, answer_index, round_id, gt_relevance):
"""
Evaluate the predict results and report metrices. Only for val split.
Parameters:
-----------
pred: ndarray of shape (n_samples, n_rounds, n_options).
answer_index: ndarray of shape (n_sample, n_rounds).
round_id: ndarray of shape (n_samples, ).
gt_relevance: ndarray of shape (n_samples, n_options).
Returns:
--------
None
"""
# Convert them to torch tensor to use visdialch.metrics
pred = torch.Tensor(pred)
answer_index = torch.Tensor(answer_index).long()
round_id = torch.Tensor(round_id).long()
gt_relevance = torch.Tensor(gt_relevance)
sparse_metrics = SparseGTMetrics()
ndcg = NDCG()
sparse_metrics.observe(pred, answer_index)
pred = pred[torch.arange(pred.size(0)), round_id - 1, :]
ndcg.observe(pred, gt_relevance)
all_metrics = {}
all_metrics.update(sparse_metrics.retrieve(reset=True))
all_metrics.update(ndcg.retrieve(reset=True))
for metric_name, metric_value in all_metrics.items():
print(f"{metric_name}: {metric_value}")
temp_train_batch[key] = batch[key].to(device)
elif key in ['ques', 'opt', 'ques_len', 'opt_len', 'ans_ind']:
temp_train_batch[key] = batch[key][:, rnd].to(device)
elif key in ['hist_len', 'hist']:
temp_train_batch[key] = batch[key][:, :rnd + 1].to(device)
else:
pass
return temp_train_batch
model.eval()
for i, batch in enumerate(val_dataloader):
batchsize = batch['img_ids'].shape[0]
rnd = 0
temp_train_batch = get_1round_batch_data(batch, rnd)
output = model(temp_train_batch).view(-1, 1, 100).detach()
for rnd in range(1, 10):
temp_train_batch = get_1round_batch_data(batch, rnd)
output = torch.cat((output, model(temp_train_batch).view(-1, 1, 100).detach()), dim=1)
sparse_metrics.observe(output, batch["ans_ind"])
if "relevance" in batch:
output = output[torch.arange(output.size(0)), batch["round_id"] - 1, :]
ndcg.observe(output.view(-1, 100), batch["relevance"].contiguous().view(-1, 100))
# if i > 5: #for debug(like the --overfit)
# break
all_metrics = {}
all_metrics.update(sparse_metrics.retrieve(reset=True))
all_metrics.update(ndcg.retrieve(reset=True))
for metric_name, metric_value in all_metrics.items():
print(f"{metric_name}: {metric_value}")
model.train()
summary_writer.add_scalar("train/loss", batch_loss, global_iteration_step)
summary_writer.add_scalar("train/lr", optimizer.param_groups[0]["lr"], global_iteration_step)
if optimizer.param_groups[0]["lr"] > config["solver"]["minimum_lr"]:
scheduler.step()
global_iteration_step += 1
# --------------------------------------------------------------------------------------------
# ON EPOCH END (checkpointing and validation)
# --------------------------------------------------------------------------------------------
checkpoint_manager.step()
# validate and report automatic metrics
if args.validate:
print(f"\nValidation after epoch {epoch}:")
for i, batch in enumerate(tqdm(val_dataloader)):
for key in batch:
batch[key] = batch[key].to(device)
with torch.no_grad():
output = model(batch)
sparse_metrics.observe(output, batch["ans_ind"])
if "gt_relevance" in batch:
output = output[torch.arange(output.size(0)), batch["round_id"] - 1, :]
ndcg.observe(output, batch["gt_relevance"])
all_metrics = {}
all_metrics.update(sparse_metrics.retrieve(reset=True))
all_metrics.update(ndcg.retrieve(reset=True))
for metric_name, metric_value in all_metrics.items():
print(f"{metric_name}: {metric_value}")
summary_writer.add_scalars("metrics", all_metrics, global_iteration_step)
class PredictionsAnalyzer:
"""
gt_index, gt_ans_relevance are found by get_dialog_by_row_index
gt_index -> 0-index
gt_round_id -> 1-index
ranks -> 1-index
ranks_json['round_id'] -> 1-index
gt_relevance -> List
# ranks_json => list of dic -> dict_keys(['image_id', 'round_id', 'ranks'])
# annotations_json list of dic -> dict_keys(['image_id', 'round_id', 'gt_relevance'])
# dialog_json list of dic -> dict_keys(['questions', 'answers', 'dialog'])
-> [''image_id', 'answer_options']
Answer options -> [3456, 677, 888] # option idx from all answers
gt_relevance -> [0.4, 0.5, ....] # relevance of options by index
Ranks json -> [100, 1, ....] # ranks of all options
Indices are same as before..but their new ranks are shown
instead of their indices being shown
Eg. [5, 6, 8, ...] means that original 0-index option is now 5th rank
"""
def __init__(self, path_val_data, dense_annotations_jsonpath,
path_images_root, model_preds_root):
self.path_val_data = path_val_data
self.dense_annotations_jsonpath = dense_annotations_jsonpath
# Ideally return the q and a here
self.read_data()
self.path_images_root = path_images_root
self.model_preds_root = model_preds_root
self.img_folder_list = self.get_img_folder_list(self.path_images_root)
self.img_map = self.get_img_map(self.img_folder_list)
self.models_list = self.get_model_type_list(self.model_preds_root)
self.gt_indices_list = [] # 0-indexed
self.gt_relevance_list = []
self.ndcg = NDCG(is_direct_ranks=True)
def get_models_list(self):
return self.models_list
def read_data(self):
self.data_val = json.load(open(self.path_val_data))
self.questions = self.data_val['data']['questions']
print("Total questions:", len(self.questions))
self.answers = self.data_val['data']['answers']
print("Total answers:", len(self.answers))
self.annotations_json = json.load(open(self.dense_annotations_jsonpath))
self.dialogs = self.data_val['data']['dialogs']
print(f"Length of all dialogs: {len(self.dialogs)}")
print(f"Length of all annotations: {len(self.annotations_json)}")
return
@staticmethod
def get_img_folder_list(path_images_root, image_folder_name="VisualDialog_val2018"):
path_visdial_val = os.path.join(path_images_root, image_folder_name)
img_folder_list = glob.glob(os.path.join(path_visdial_val, '*'))
print("Total images in folder:", len(img_folder_list))
return img_folder_list
@staticmethod
def get_model_type_list(model_preds_root):
model_folder_list = [os.path.basename(x) for x in glob.glob(os.path.join(model_preds_root, '*'))]
print("Total models in folder:", len(model_folder_list))
return model_folder_list
@staticmethod
def json_load(file_path):
with open(file_path, "r") as fb:
data = json.load(fb)
return data
@staticmethod
def convert_list_json_dic(ranks_json):
image_ranks_dic = {}
for i in range(len(ranks_json)):
image_ranks_dic[ranks_json[i]["image_id"]] = ranks_json[i]
return image_ranks_dic
@staticmethod
def image_id_from_path(image_path):
"""Given a path to an image, return its id.
Parameters
----------
image_path : str
Path to image, e.g.: coco_train2014/COCO_train2014/000000123456.jpg
img_name = "VisualDialog_val2018_000000254080.jpg"
Returns
-------
int
Corresponding image id (123456)
"""
return int(image_path.split("/")[-1][-16:-4])
def get_img_map(self, img_folder_list):
img_map = dict()
for img_path in img_folder_list:
img_id = self.image_id_from_path(img_path)
img_map[img_id] = img_path
return img_map
def show_img(self, img_id):
img_path = self.img_map[img_id]
print("Reading image from: ", img_path)
plt.imshow(plt.imread(img_path))
def get_both_phase_ranks(self, model_type,
ranks_phase1_file="ranks_val_12_crowdsourced.json",
ranks_finetune_file="ranks_val_best_ndcg_crowdsourced.json"):
"""
:param model_type:
:param ranks_phase1_file:
:param ranks_finetune_file:
:return:
"""
model_rank_phase1_path = Path(self.model_preds_root, model_type, ranks_phase1_file)
model_rank_finetune_path = Path(self.model_preds_root, model_type, ranks_finetune_file)
# list of dic -> dict_keys(['image_id', 'round_id', 'ranks'])
ranks_phase1_json = self.json_load(model_rank_phase1_path)
ranks_finetune_json = self.json_load(model_rank_finetune_path)
return ranks_phase1_json, ranks_finetune_json
def subset_val_ranks_with_dense_annotation(self, ranks_json, top_k=5):
"""
this is because val ranks consists of 10 turns
:param ranks_json:
:param top_k:
:return:
"""
rank_dense_list = []
relevance_dic = {}
index_dic = {}
gt_results_index_dic = {}
# gt_results_relevance_dic = {}
gt_indices_list = [] # 0-indexed
gt_relevance_list = []
dialogs = self.data_val['data']['dialogs']
for i in range(len(self.annotations_json)):
# They will be in same order by image_id
round_id = self.annotations_json[i]['round_id'] - 1 # 0-indexing
index_for_ranks_json = i * 10 + round_id # for each image: 10
assert ranks_json[index_for_ranks_json]['round_id'] == round_id + 1 # Check with 1-indexing
assert ranks_json[index_for_ranks_json]['image_id'] == self.annotations_json[i]['image_id']
gt_relevance = self.annotations_json[i]['gt_relevance']
rank_dense_list.append(ranks_json[index_for_ranks_json])
ranks = ranks_json[index_for_ranks_json]['ranks']
relevance_sum = 0
image_id = ranks_json[index_for_ranks_json]['image_id']
# To actually have the indices_list and relevance list before hand
assert image_id == dialogs[i]['image_id']
gt_index = dialogs[i]['dialog'][round_id]['gt_index']
# round_id already 0-index gt_index also 0-indexed
gt_ans_relevance = gt_relevance[gt_index]
gt_indices_list.append(gt_index)
gt_relevance_list.append(gt_ans_relevance)
# We need to find rank of (gt_index + 1) - coz ranks are 1-indexed
pred_index = ranks.index(gt_index + 1) # maintaining 0-index
if pred_index in gt_results_index_dic:
# gt_results_index_dic[pred_index].append(image_id)
gt_results_index_dic[pred_index].append(i)
else:
# gt_results_index_dic[pred_index] = [image_id]
gt_results_index_dic[pred_index] = [i]
for j in range(top_k):
relevance_sum += gt_relevance[ranks[j] - 1] # 0-indexing
# We keep a list of relevance sum - kind of proxy to get the best scores
if relevance_sum in relevance_dic:
relevance_dic[relevance_sum].append(image_id)
index_dic[relevance_sum].append(i)
else:
relevance_dic[relevance_sum] = [image_id]
index_dic[relevance_sum] = [i]
# Re-assign
self.gt_indices_list = gt_indices_list
self.gt_relevance_list = gt_relevance_list
return rank_dense_list, relevance_dic, index_dic, gt_results_index_dic
def print_dialog(self, dialog, gt_round_id, gt_ans_relevance, answer_options):
"""
dialog -> per image
gt_round_id -> should be 1-index
"""
print("\n")
print("Dialog: ")
print("\n")
for round_indx in range(gt_round_id - 1):
print("Q{}".format(round_indx + 1), f"{self.questions[dialog[round_indx]['question']].capitalize()}?")
print("A{}".format(round_indx + 1), f"{self.answers[dialog[round_indx]['answer']].capitalize()}.")
print("\n")
print("Question {}: ".format(gt_round_id), f"{self.questions[dialog[gt_round_id - 1]['question']].capitalize()}?")
print("\n")
print("GT answer: ", f"{self.answers[dialog[gt_round_id - 1]['answer']].capitalize()}.")
gt_ans_index = dialog[gt_round_id - 1]['answer']
found_ans_index = answer_options.index(gt_ans_index) # returns 0-index
# print("GT index: ", dialog[gt_round_id - 1]['gt_index'])
gt_index = dialog[gt_round_id - 1]['gt_index']
assert found_ans_index == gt_index
print("GT relevance: ", gt_ans_relevance)
print("\n")
def get_dialog_by_row_index(self, row_index, is_print):
"""
:param row_index: defines the whole dialog (not turn)
:return:
"""
caption = self.data_val['data']['dialogs'][row_index]['caption']
dialog = self.data_val['data']['dialogs'][row_index]['dialog']
image_id = self.data_val['data']['dialogs'][row_index]['image_id']
# This is for turn
dense_annotations = self.annotations_json[row_index]
# print(dense_annotations.keys())
gt_round_id = dense_annotations["round_id"] # 1-index
gt_image_id = dense_annotations["image_id"]
assert gt_image_id == image_id
gt_relevance = dense_annotations["gt_relevance"]
assert len(gt_relevance) == 100
gt_index = dialog[gt_round_id - 1]['gt_index']
gt_ans_relevance = gt_relevance[gt_index]
answer_options = dialog[gt_round_id - 1]["answer_options"]
if is_print:
print(caption)
# self.show_img(image_id)
self.print_dialog(dialog, gt_round_id, gt_ans_relevance, answer_options)
non_zero_relevant_ans = np.count_nonzero(gt_relevance)
print("Number of answers with non-zero relevance: ", non_zero_relevant_ans)
return answer_options, gt_index, gt_relevance, gt_round_id
def print_top_k_preds(self, ranks_model, gt_index, gt_relevance,
answer_options, phase, top_k=5):
"""
gt_index -> should be 0-index
"""
print("\n")
print(f"{phase} Phase: ")
# Get rank of gt_index
# pred_index = ranks_model.index(gt_index) + 1 # to convert to 1-index
# print("GT answer predicted at: ", pred_index)
# print("GT answer is: ", self.answers[answer_options[ranks_model[pred_index-1]]])
# reverse_ranks = ranks_model[::-1]
# print(reverse_ranks)
# print(ranks_model)
# print(self.get_max_index(gt_relevance))
print("GT predicted rank: ", ranks_model[gt_index] +1) # gt rank and ranks both 0-index
# indices are same. but their ranks are shown now. instead of option number being shown.
indices_list = sorted(range(len(ranks_model)), key=lambda i: ranks_model[i])[:top_k]
self.get_ndcg_value_wrapper(ranks=ranks_model, gt_relevance=gt_relevance)
for i in range(top_k):
print("Relevance:", gt_relevance[indices_list[i]], "Answer:", f"{self.answers[answer_options[indices_list[i]]].capitalize()}.")
print("\n")
def get_ndcg_value_wrapper(self, ranks: List,
gt_relevance : List):
"""
:param ranks: list
:param gt_relevance: list
:return:
"""
# (batch_size, num_options)
ranks = torch.tensor(ranks).float()
gt_relevance = torch.tensor(gt_relevance).float()
# If individual sample, we need to add 0-dim
if len(ranks.size()) == 1:
ranks = ranks.unsqueeze(0)
gt_relevance = gt_relevance.unsqueeze(0)
self.ndcg.observe(ranks, gt_relevance)
value = self.ndcg.retrieve(reset=True)["ndcg"]
print(f"NDCG: {round(value*100,2)}")
return value
def print_top_k_preds_wrapper(self, ranks_phase1_json, ranks_finetune_json,
gt_index, gt_relevance, answer_options,
row_index, gt_round_id, top_k=5):
ranks_phase1 = ranks_phase1_json[row_index]["ranks"]
ranks_finetune = ranks_finetune_json[row_index]["ranks"]
assert ranks_phase1_json[row_index]["round_id"] == gt_round_id
# Ranks are 1 indexed - shifting to 0
ranks_phase1 = [rank - 1 for rank in ranks_phase1]
ranks_finetune = [rank - 1 for rank in ranks_finetune]
self.print_top_k_preds(ranks_phase1, gt_index, gt_relevance,
answer_options, "Spare Annotation", top_k=top_k)
self.print_top_k_preds(ranks_finetune, gt_index, gt_relevance,
answer_options, "Curriculum Finetuning", top_k=top_k)
# Main api open to call
def get_analysis(self, model_type, top_k=5, row_index=1, is_print=False):
ranks_phase1_json, ranks_finetune_json = self.get_both_phase_ranks(model_type)
# print(f"Length of ranks phase 1 json was earlier: {len(ranks_phase1_json)}")
# print(f"Length of ranks finetune json was earlier: {len(ranks_finetune_json)}")
ranks_phase1_json, relevance_dic_phase1, index_dic_phase1,\
gt_results_index_dic_phase1 = self.subset_val_ranks_with_dense_annotation(
ranks_phase1_json, top_k=top_k)
# print(f"After subset Length of ranks phase 1 json : {len(ranks_phase1_json)}")
ranks_finetune_json, relevance_dic_finetune, \
index_dic_finetune, gt_results_index_dic_finetune = self.subset_val_ranks_with_dense_annotation(
ranks_finetune_json, top_k=top_k)
# print(f"After subset Length of ranks finetune json : {len(ranks_phase1_json)}")
# For identifying row_index
# print(relevance_dic_phase1)
# print(relevance_dic_finetune)
# print(index_dic_finetune)
answer_options, gt_index, gt_relevance, gt_round_id = self.get_dialog_by_row_index(row_index, is_print)
self.print_top_k_preds_wrapper(ranks_phase1_json, ranks_finetune_json,
gt_index, gt_relevance, answer_options,
row_index, gt_round_id, top_k=top_k)
# return relevance_dic_phase1, relevance_dic_finetune, index_dic_finetune
return
# Return only once to find subsets
def get_dic_models(self, model_type, top_k=5):
ranks_phase1_json, ranks_finetune_json = self.get_both_phase_ranks(model_type)
ranks_phase1_json, relevance_dic_phase1, \
index_dic_phase1, gt_results_index_dic_phase1\
= self.subset_val_ranks_with_dense_annotation(
ranks_phase1_json, top_k=top_k)
ranks_finetune_json, relevance_dic_finetune, \
index_dic_finetune, gt_results_index_dic_finetune = self.subset_val_ranks_with_dense_annotation(
ranks_finetune_json, top_k=top_k)
return relevance_dic_phase1, relevance_dic_finetune, index_dic_phase1, \
index_dic_finetune, gt_results_index_dic_phase1, \
gt_results_index_dic_finetune
@staticmethod
def get_max_index(values):
# Returns 0-index
_index = values.index(max(values))
return _index
def list_ans_opts(self, answer_options, gt_relevance, gt_index, num_ans_opts=5):
"""
To list answer options for the task
:param answer_options:
:param num_ans_opts:
:return:
"""
print_ans_opt_list = []
print_relevance = []
for ans_opt in range(num_ans_opts-1):
print_ans_opt_list.append(self.answers[answer_options[ans_opt]])
print_relevance.append(gt_relevance[ans_opt])
# print(self.answers[answer_options[ans_opt]])
# print(gt_relevance[ans_opt])
# One we will print for max index
max_score_index = self.get_max_index(gt_relevance) # 0-index
print_ans_opt_list.append(self.answers[answer_options[max_score_index]])
print_relevance.append(gt_relevance[max_score_index])
print_ans_opt_list.append(self.answers[answer_options[gt_index]])
print_relevance.append(gt_relevance[gt_index])
print("Answers:")
self.print_line_by_line(print_ans_opt_list)
print("Relevance:")
self.print_line_by_line(print_relevance)
print("Last one is gt")
# print(print_ans_opt_list)
# print(print_relevance)
# print(self.answers[answer_options[max_score_index]])
# print(gt_relevance[max_score_index])
@staticmethod
def print_line_by_line(mylist):
for elem in mylist:
print(elem)
class SubsetComplementVisDial:
"""
We will be subsetting actual dataset based on subset image ids
"""
def __init__(self, config):
super().__init__()
self.ndcg = NDCG(is_direct_ranks=True)
# We are calculating NDCG directly based on ranks
# self.path_val_data = config.path_val_data
self.dense_annotations_jsonpath = config.dense_annotations_jsonpath
self.model_preds_root = config.model_preds_root
self.models_list = self.get_model_type_list(self.model_preds_root)
self.annotations_json = json.load(open(
self.dense_annotations_jsonpath))
self.hist_info_images = [
257366, 425477, 191097, 552399, 12468, 458949, 109735, 311793,
437200, 355853, 98849, 57743, 83289, 488471, 446567, 196905,
308846, 328336, 289233, 52156, 366462, 511748, 457675, 518811,
413085, 432039, 531270, 430580, 293582, 544148, 80366, 179366,
150236, 400960, 10424, 451398, 498340, 268914, 384171, 172461,
387266, 214227, 555578, 181772, 149373, 251385, 407878, 574545,
544827, 120559, 19299, 73638, 496822, 204195, 97073, 209447, 53433,
403234, 524006, 178300, 376460, 570468, 292100, 227006, 170315,
456824, 525726, 179064, 98879, 558975, 193521, 377823, 449230,
44468, 573552, 288308, 237956, 69538, 250654, 439842, 146314,
458818, 122826, 33976, 322815, 239030, 209271, 560666, 361734,
225491, 27366, 29060, 191186, 394073, 120870, 580183, 111013
]
self.subset_type = "_complement"
def extract_ndcg(self, model_type: str, num_samples: int = 97):
"""
:param model_type: for which we want to extract
:return:
"""
try:
ranks_phase1_file = f"ranks_val_12.json"
ranks_finetune_file = f"ranks_val_best_ndcg.json"
ranks_phase1_json, ranks_finetune_json = self.get_both_phase_ranks(
model_type,
ranks_phase1_file=ranks_phase1_file,
ranks_finetune_file=ranks_finetune_file)
except:
print(
f"For Model {model_type}, we are using 11 as the ckpt based on ndcg"
)
ranks_phase1_file = f"ranks_val_11.json"
ranks_finetune_file = f"ranks_val_best_ndcg.json"
ranks_phase1_json, ranks_finetune_json = self.get_both_phase_ranks(
model_type,
ranks_phase1_file=ranks_phase1_file,
ranks_finetune_file=ranks_finetune_file)
# ranks_phase1_json, ranks_finetune_json = self.get_both_phase_ranks(model_type)
# Remove all examples considered as hist info
ranks_phase1_json, new_annotation_list = self.subset_val_ranks_with_dense_annotation(
ranks_phase1_json)
ranks_finetune_json, new_annotation_list = self.subset_val_ranks_with_dense_annotation(
ranks_finetune_json)
# Sample all elements here
indices_list = random.sample(range(len(ranks_phase1_json)),
k=num_samples)
ranks_phase1_json = [ranks_phase1_json[i] for i in indices_list]
ranks_finetune_json = [ranks_finetune_json[i] for i in indices_list]
new_annotation_list = [new_annotation_list[i] for i in indices_list]
assert len(ranks_phase1_json) == len(ranks_finetune_json) == len(
new_annotation_list)
gt_relevance_list = []
ranks_list_phase1 = []
ranks_list_finetune = []
ndcg_list_phase1 = []
ndcg_list_finetune = []
for indx in range(len(ranks_phase1_json)):
ranks_phase1 = ranks_phase1_json[indx]["ranks"]
ranks_finetune = ranks_finetune_json[indx]["ranks"]
gt_relevance = new_annotation_list[indx]['gt_relevance']
# Assert if we are doing for same round ids
assert ranks_phase1_json[indx]['round_id'] == new_annotation_list[
indx]['round_id']
assert ranks_finetune_json[indx][
'round_id'] == new_annotation_list[indx]['round_id']
ndcg_sample_phase1 = self.get_ndcg_value_wrapper(
ranks_phase1, gt_relevance)
ndcg_sample_finetune = self.get_ndcg_value_wrapper(
ranks_finetune, gt_relevance)
# Maintain the list for individual samples
ndcg_list_phase1.append(ndcg_sample_phase1)
ndcg_list_finetune.append(ndcg_sample_finetune)
# For whole set - for verification
ranks_list_phase1.append(ranks_phase1)
ranks_list_finetune.append(ranks_finetune)
gt_relevance_list.append(gt_relevance)
# For whole val set, for verification..Not saving them!
# ndcg_sample_phase1 = self.get_ndcg_value_wrapper(ranks_list_phase1, gt_relevance_list)
# ndcg_sample_finetune = self.get_ndcg_value_wrapper(ranks_list_finetune, gt_relevance_list)
# print(f"NDCG for the whole set for {model_type} (phase1): ", ndcg_sample_phase1*100)
# print(f"NDCG for the whole set for {model_type}(finetune)", ndcg_sample_finetune*100)
ndcg_write_path = self._get_ndcg_path(self.model_preds_root,
model_type,
phase="sparse",
subset_type=self.subset_type)
print(f"Saving as {ndcg_write_path}")
self.write_list_to_file(ndcg_write_path, ndcg_list_phase1)
ndcg_write_path = self._get_ndcg_path(self.model_preds_root,
model_type,
phase="finetune",
subset_type=self.subset_type)
print(f"Saving as {ndcg_write_path}")
self.write_list_to_file(ndcg_write_path, ndcg_list_finetune)
def get_ndcg_value_wrapper(self, ranks: List, gt_relevance: List):
"""
:param ranks: list
:param gt_relevance: list
:return:
"""
# (batch_size, num_options)
ranks = torch.tensor(ranks).float()
gt_relevance = torch.tensor(gt_relevance).float()
# If individual sample, we need to add 0-dim
if len(ranks.size()) == 1:
ranks = ranks.unsqueeze(0)
gt_relevance = gt_relevance.unsqueeze(0)
self.ndcg.observe(ranks, gt_relevance)
value = self.ndcg.retrieve(reset=True)["ndcg"]
return value
# SA: TODO - Notice ranks file changed here
def get_both_phase_ranks(self,
model_type,
ranks_phase1_file="ranks_val_12.json",
ranks_finetune_file="ranks_val_best_ndcg.json"):
"""
:param model_type:
:param ranks_phase1_file:
:param ranks_finetune_file:
:return:
"""
model_rank_phase1_path = Path(self.model_preds_root, model_type,
ranks_phase1_file)
model_rank_finetune_path = Path(self.model_preds_root, model_type,
ranks_finetune_file)
# list of dic -> dict_keys(['image_id', 'round_id', 'ranks'])
ranks_phase1_json = self.json_load(model_rank_phase1_path)
ranks_finetune_json = self.json_load(model_rank_finetune_path)
return ranks_phase1_json, ranks_finetune_json
def subset_val_ranks_with_dense_annotation(self, ranks_json):
"""
this is because val ranks consists of 10 turns
:param ranks_json:
:param top_k:
:return:
"""
rank_dense_list = []
new_annotation_list = []
for i in range(len(self.annotations_json)):
# They will be in same order by image_id
round_id = self.annotations_json[i]['round_id'] - 1 # 0-indexing
index_for_ranks_json = i * 10 + round_id # for each image: 10
assert ranks_json[index_for_ranks_json][
'round_id'] == round_id + 1 # Check with 1-indexing
assert ranks_json[index_for_ranks_json][
'image_id'] == self.annotations_json[i]['image_id']
# Subset the data here
image_id = self.annotations_json[i]['image_id']
if image_id not in self.hist_info_images:
rank_dense_list.append(ranks_json[index_for_ranks_json])
new_annotation_list.append(self.annotations_json[i])
return rank_dense_list, new_annotation_list
@staticmethod
def write_list_to_file(filepath, write_list) -> None:
with open(filepath, 'w') as file_handler:
for item in write_list:
file_handler.write("{}\n".format(item))
# outfile.write("\n".join(itemlist))
return
@staticmethod
def json_load(file_path):
with open(file_path, "r") as fb:
data = json.load(fb)
return data
@staticmethod
def get_model_type_list(model_preds_root) -> List:
model_folder_list = [
os.path.basename(x)
for x in glob.glob(os.path.join(model_preds_root, '*'))
]
print("Total models in folder:", len(model_folder_list))
return model_folder_list
@staticmethod
def _get_ndcg_path(model_root: str,
model_type: str,
phase: str,
subset_type: str = '_complement',
ext: str = 'txt') -> str:
json_path = f"{model_root}/{model_type}/ndcg_{phase}_{model_type}{subset_type}.{ext}"
return json_path
def train(config,
args,
dataloader_dic,
device,
finetune: bool = False,
load_pthpath: str = "",
finetune_regression: bool = False,
dense_scratch_train: bool = False,
dense_annotation_type: str = "default"):
"""
:param config:
:param args:
:param dataloader_dic:
:param device:
:param finetune:
:param load_pthpath:
:param finetune_regression:
:param dense_scratch_train: when we want to start training only on 2000 annotations
:param dense_annotation_type: default
:return:
"""
# =============================================================================
# SETUP BEFORE TRAINING LOOP
# =============================================================================
train_dataset = dataloader_dic["train_dataset"]
train_dataloader = dataloader_dic["train_dataloader"]
val_dataloader = dataloader_dic["val_dataloader"]
val_dataset = dataloader_dic["val_dataset"]
model = get_model(config, args, train_dataset, device)
if finetune and not dense_scratch_train:
assert load_pthpath != "", "Please provide a path" \
" for pre-trained model before " \
"starting fine tuning"
print(f"\n Begin Finetuning:")
optimizer, scheduler, iterations, lr_scheduler_type = get_solver(
config, args, train_dataset, val_dataset, model, finetune=finetune)
start_time = datetime.datetime.strftime(datetime.datetime.utcnow(),
'%d-%b-%Y-%H:%M:%S')
if args.save_dirpath == 'checkpoints/':
args.save_dirpath += '%s+%s/%s' % (
config["model"]["encoder"], config["model"]["decoder"], start_time)
summary_writer = SummaryWriter(log_dir=args.save_dirpath)
checkpoint_manager = CheckpointManager(model,
optimizer,
args.save_dirpath,
config=config)
sparse_metrics = SparseGTMetrics()
ndcg = NDCG()
best_val_loss = np.inf # SA: initially loss can be any number
best_val_ndcg = 0.0
# If loading from checkpoint, adjust start epoch and load parameters.
# SA: 1. if finetuning -> load from saved model
# 2. train -> default load_pthpath = ""
# 3. else load pthpath
if (not finetune and load_pthpath == "") or dense_scratch_train:
start_epoch = 1
else:
# "path/to/checkpoint_xx.pth" -> xx
### To cater model finetuning from models with "best_ndcg" checkpoint
try:
start_epoch = int(load_pthpath.split("_")[-1][:-4]) + 1
except:
start_epoch = 1
model_state_dict, optimizer_state_dict = load_checkpoint(load_pthpath)
# SA: updating last epoch
checkpoint_manager.update_last_epoch(start_epoch)
if isinstance(model, nn.DataParallel):
model.module.load_state_dict(model_state_dict)
else:
model.load_state_dict(model_state_dict)
# SA: for finetuning optimizer should start from its learning rate
if not finetune:
optimizer.load_state_dict(optimizer_state_dict)
else:
print("Optimizer not loaded. Different optimizer for finetuning.")
print("Loaded model from {}".format(load_pthpath))
# =============================================================================
# TRAINING LOOP
# =============================================================================
# Forever increasing counter to keep track of iterations (for tensorboard log).
global_iteration_step = (start_epoch - 1) * iterations
running_loss = 0.0 # New
train_begin = datetime.datetime.utcnow() # New
if finetune:
end_epoch = start_epoch + config["solver"]["num_epochs_curriculum"] - 1
if finetune_regression:
# criterion = nn.MSELoss(reduction='mean')
# criterion = nn.KLDivLoss(reduction='mean')
criterion = nn.MultiLabelSoftMarginLoss()
else:
end_epoch = config["solver"]["num_epochs"]
# SA: normal training
criterion = get_loss_criterion(config, train_dataset)
# SA: end_epoch + 1 => for loop also doing last epoch
for epoch in range(start_epoch, end_epoch + 1):
# -------------------------------------------------------------------------
# ON EPOCH START (combine dataloaders if training on train + val)
# -------------------------------------------------------------------------
if config["solver"]["training_splits"] == "trainval":
combined_dataloader = itertools.chain(train_dataloader,
val_dataloader)
else:
combined_dataloader = itertools.chain(train_dataloader)
print(f"\nTraining for epoch {epoch}:")
for i, batch in enumerate(tqdm(combined_dataloader)):
for key in batch:
batch[key] = batch[key].to(device)
optimizer.zero_grad()
output = model(batch)
if finetune:
target = batch["gt_relevance"]
# Same as for ndcg validation, only one round is present
output = output[torch.arange(output.size(0)),
batch["round_id"] - 1, :]
# SA: todo regression loss
if finetune_regression:
batch_loss = mse_loss(output, target, criterion)
else:
batch_loss = compute_ndcg_type_loss(output, target)
else:
batch_loss = get_batch_criterion_loss_value(
config, batch, criterion, output)
batch_loss.backward()
optimizer.step()
# --------------------------------------------------------------------
# update running loss and decay learning rates
# --------------------------------------------------------------------
if running_loss > 0.0:
running_loss = 0.95 * running_loss + 0.05 * batch_loss.item()
else:
running_loss = batch_loss.item()
# SA: lambda_lr was configured to reduce lr after milestone epochs
if lr_scheduler_type == "lambda_lr":
scheduler.step(global_iteration_step)
global_iteration_step += 1
if global_iteration_step % 100 == 0:
# print current time, running average, learning rate, iteration, epoch
print(
"[{}][Epoch: {:3d}][Iter: {:6d}][Loss: {:6f}][lr: {:8f}]".
format(datetime.datetime.utcnow() - train_begin, epoch,
global_iteration_step, running_loss,
optimizer.param_groups[0]['lr']))
# tensorboardX
summary_writer.add_scalar("train/loss", batch_loss,
global_iteration_step)
summary_writer.add_scalar("train/lr",
optimizer.param_groups[0]["lr"],
global_iteration_step)
torch.cuda.empty_cache()
# -------------------------------------------------------------------------
# ON EPOCH END (checkpointing and validation)
# -------------------------------------------------------------------------
if not finetune:
checkpoint_manager.step(epoch=epoch)
else:
print("Validating before checkpointing.")
# SA: ideally another function: too much work
# Validate and report automatic metrics.
if args.validate:
# Switch dropout, batchnorm etc to the correct mode.
model.eval()
val_loss = 0
print(f"\nValidation after epoch {epoch}:")
for i, batch in enumerate(tqdm(val_dataloader)):
for key in batch:
batch[key] = batch[key].to(device)
with torch.no_grad():
output = model(batch)
if finetune:
target = batch["gt_relevance"]
# Same as for ndcg validation, only one round is present
out_ndcg = output[torch.arange(output.size(0)),
batch["round_id"] - 1, :]
# SA: todo regression loss
if finetune_regression:
batch_loss = mse_loss(out_ndcg, target, criterion)
else:
batch_loss = compute_ndcg_type_loss(
out_ndcg, target)
else:
batch_loss = get_batch_criterion_loss_value(
config, batch, criterion, output)
val_loss += batch_loss.item()
sparse_metrics.observe(output, batch["ans_ind"])
if "gt_relevance" in batch:
output = output[torch.arange(output.size(0)),
batch["round_id"] - 1, :]
ndcg.observe(output, batch["gt_relevance"])
all_metrics = {}
all_metrics.update(sparse_metrics.retrieve(reset=True))
all_metrics.update(ndcg.retrieve(reset=True))
for metric_name, metric_value in all_metrics.items():
print(f"{metric_name}: {metric_value}")
summary_writer.add_scalars("metrics", all_metrics,
global_iteration_step)
model.train()
torch.cuda.empty_cache()
val_loss = val_loss / len(val_dataloader)
print(f"Validation loss for {epoch} epoch is {val_loss}")
print(f"Validation loss for batch is {batch_loss}")
summary_writer.add_scalar("val/loss", batch_loss,
global_iteration_step)
if val_loss < best_val_loss:
print(f" Best model found at {epoch} epoch! Saving now.")
best_val_loss = val_loss
if dense_annotation_type == "default":
checkpoint_manager.save_best()
else:
print(f" Not saving the model at {epoch} epoch!")
# SA: Saving the best model both for loss and ndcg now
val_ndcg = all_metrics["ndcg"]
if val_ndcg > best_val_ndcg:
print(f" Best ndcg model found at {epoch} epoch! Saving now.")
best_val_ndcg = val_ndcg
if dense_annotation_type == "default":
checkpoint_manager.save_best(ckpt_name="best_ndcg")
else:
# SA: trying for dense annotations
ckpt_name = f"best_ndcg_annotation_{dense_annotation_type}"
checkpoint_manager.save_best(ckpt_name=ckpt_name)
else:
print(f" Not saving the model at {epoch} epoch!")
# SA: "reduce_lr_on_plateau" works only with validate for now
if lr_scheduler_type == "reduce_lr_on_plateau":
# scheduler.step(val_loss)
# SA: # Loss should decrease while ndcg should increase!
# can also change the mode in LR reduce on plateau to max
scheduler.step(-1 * val_ndcg)
class NDCGForRanks:
def __init__(self, config):
super().__init__()
self.ndcg = NDCG(is_direct_ranks=True)
# We are calculating NDCG directly based on ranks
# self.path_val_data = config.path_val_data
self.dense_annotations_jsonpath = config.dense_annotations_jsonpath
self.model_preds_root = config.model_preds_root
self.models_list = self.get_model_type_list(self.model_preds_root)
self.annotations_json = json.load(open(
self.dense_annotations_jsonpath))
self.subset_type = config.subset_type
def extract_ndcg(self, model_type: str):
"""
:param model_type: for which we want to extract
:return:
"""
try:
ranks_phase1_file = f"ranks_val_12{self.subset_type}.json"
ranks_finetune_file = f"ranks_val_best_ndcg{self.subset_type}.json"
ranks_phase1_json, ranks_finetune_json = self.get_both_phase_ranks(
model_type,
ranks_phase1_file=ranks_phase1_file,
ranks_finetune_file=ranks_finetune_file)
except:
print(
f"For Model {model_type}, we are using 11 as the ckpt based on ndcg"
)
ranks_phase1_file = f"ranks_val_11{self.subset_type}.json"
ranks_finetune_file = f"ranks_val_best_ndcg{self.subset_type}.json"
ranks_phase1_json, ranks_finetune_json = self.get_both_phase_ranks(
model_type,
ranks_phase1_file=ranks_phase1_file,
ranks_finetune_file=ranks_finetune_file)
ranks_phase1_json = self.subset_val_ranks_with_dense_annotation(
ranks_phase1_json)
ranks_finetune_json = self.subset_val_ranks_with_dense_annotation(
ranks_finetune_json)
assert len(ranks_phase1_json) == len(ranks_finetune_json) == len(
self.annotations_json)
gt_relevance_list = []
ranks_list_phase1 = []
ranks_list_finetune = []
ndcg_list_phase1 = []
ndcg_list_finetune = []
for indx in range(len(ranks_phase1_json)):
ranks_phase1 = ranks_phase1_json[indx]["ranks"]
ranks_finetune = ranks_finetune_json[indx]["ranks"]
gt_relevance = self.annotations_json[indx]['gt_relevance']
# Assert if we are doing for same round ids
assert ranks_phase1_json[indx][
'round_id'] == self.annotations_json[indx]['round_id']
assert ranks_finetune_json[indx][
'round_id'] == self.annotations_json[indx]['round_id']
ndcg_sample_phase1 = self.get_ndcg_value_wrapper(
ranks_phase1, gt_relevance)
ndcg_sample_finetune = self.get_ndcg_value_wrapper(
ranks_finetune, gt_relevance)
# Maintain the list for individual samples
ndcg_list_phase1.append(ndcg_sample_phase1)
ndcg_list_finetune.append(ndcg_sample_finetune)
# For whole set - for verification
ranks_list_phase1.append(ranks_phase1)
ranks_list_finetune.append(ranks_finetune)
gt_relevance_list.append(gt_relevance)
# For whole val set, for verification..Not saving them!
ndcg_sample_phase1 = self.get_ndcg_value_wrapper(
ranks_list_phase1, gt_relevance_list)
ndcg_sample_finetune = self.get_ndcg_value_wrapper(
ranks_list_finetune, gt_relevance_list)
print(f"NDCG for the whole set for {model_type} (phase1): ",
round(ndcg_sample_phase1 * 100, 2))
print(f"NDCG for the whole set for {model_type}(finetune)",
round(ndcg_sample_finetune * 100, 2))
ndcg_write_path = self._get_ndcg_path(self.model_preds_root,
model_type,
phase="sparse",
subset_type=self.subset_type)
print(f"Saving as {ndcg_write_path}")
self.write_list_to_file(ndcg_write_path, ndcg_list_phase1)
ndcg_write_path = self._get_ndcg_path(self.model_preds_root,
model_type,
phase="finetune",
subset_type=self.subset_type)
print(f"Saving as {ndcg_write_path}")
self.write_list_to_file(ndcg_write_path, ndcg_list_finetune)
def get_ndcg_value_wrapper(self, ranks: List, gt_relevance: List):
"""
:param ranks: list
:param gt_relevance: list
:return:
"""
# (batch_size, num_options)
ranks = torch.tensor(ranks).float()
gt_relevance = torch.tensor(gt_relevance).float()
# If individual sample, we need to add 0-dim
if len(ranks.size()) == 1:
ranks = ranks.unsqueeze(0)
gt_relevance = gt_relevance.unsqueeze(0)
self.ndcg.observe(ranks, gt_relevance)
value = self.ndcg.retrieve(reset=True)["ndcg"]
return value
def get_both_phase_ranks(
self,
model_type,
ranks_phase1_file="ranks_val_12_crowdsourced.json",
ranks_finetune_file="ranks_val_best_ndcg_crowdsourced.json"):
"""
:param model_type:
:param ranks_phase1_file:
:param ranks_finetune_file:
:return:
"""
model_rank_phase1_path = Path(self.model_preds_root, model_type,
ranks_phase1_file)
model_rank_finetune_path = Path(self.model_preds_root, model_type,
ranks_finetune_file)
# list of dic -> dict_keys(['image_id', 'round_id', 'ranks'])
ranks_phase1_json = self.json_load(model_rank_phase1_path)
ranks_finetune_json = self.json_load(model_rank_finetune_path)
return ranks_phase1_json, ranks_finetune_json
def subset_val_ranks_with_dense_annotation(self, ranks_json):
"""
this is because val ranks consists of 10 turns
:param ranks_json:
:param top_k:
:return:
"""
rank_dense_list = []
# gt_indices_list = [] # 0-indexed
# gt_relevance_list = []
# dialogs = self.data_val['data']['dialogs']
for i in range(len(self.annotations_json)):
# They will be in same order by image_id
round_id = self.annotations_json[i]['round_id'] - 1 # 0-indexing
index_for_ranks_json = i * 10 + round_id # for each image: 10
assert ranks_json[index_for_ranks_json][
'round_id'] == round_id + 1 # Check with 1-indexing
assert ranks_json[index_for_ranks_json][
'image_id'] == self.annotations_json[i]['image_id']
rank_dense_list.append(ranks_json[index_for_ranks_json])
# ranks = ranks_json[index_for_ranks_json]['ranks']
# image_id = ranks_json[index_for_ranks_json]['image_id']
# gt_relevance = self.annotations_json[i]['gt_relevance']
# # To actually have the indices_list and relevance list before hand
# assert image_id == dialogs[i]['image_id']
# gt_index = dialogs[i]['dialog'][round_id]['gt_index']
# # round_id already 0-index gt_index also 0-indexed
# gt_ans_relevance = gt_relevance[gt_index]
# gt_indices_list.append(gt_index)
# gt_relevance_list.append(gt_ans_relevance)
# # We need to find rank of (gt_index + 1) - coz ranks are 1-indexed
# pred_index = ranks.index(gt_index + 1) # maintaining 0-index
return rank_dense_list
@staticmethod
def write_list_to_file(filepath, write_list) -> None:
with open(filepath, 'w') as file_handler:
for item in write_list:
file_handler.write("{}\n".format(item))
# outfile.write("\n".join(itemlist))
return
@staticmethod
def json_load(file_path):
with open(file_path, "r") as fb:
data = json.load(fb)
return data
@staticmethod
def get_model_type_list(model_preds_root) -> List:
model_folder_list = [
os.path.basename(x)
for x in glob.glob(os.path.join(model_preds_root, '*'))
]
print("Total models in folder:", len(model_folder_list))
return model_folder_list
@staticmethod
def _get_ndcg_path(model_root: str,
model_type: str,
phase: str,
subset_type: str = '_crowdsourced',
ext: str = 'txt') -> str:
json_path = f"{model_root}/{model_type}/ndcg_{phase}_{model_type}{subset_type}.{ext}"
return json_path
