def rank_average(preds, temp=1):
"""
Ensemble several model's predicts by average their predicted ranks.
Parameters:
-----------
preds: ndarray of shape (n_models, n_samples, n_rounds, n_options).
Several model's predict results.
Returns:
--------
final_pred: ndarray of shape (n_sample, n_rounds, n_options).
The ensembled result.
"""
ranks = np.array(
[scores_to_ranks(torch.tensor(pred)).cpu().numpy() for pred in preds])
ranks = (ranks - ranks.min()) / (ranks.max() - ranks.min())
return 1.0 - (ranks**temp).mean(axis=0)
def write_to_json(pred, image_id, round_id, json_path, split):
"""
Write predict results to json file.
Parameters:
-----------
pred: ndarray of shape (n_samples, n_rounds, n_options).
image_id: ndarray of shape (n_sample, ).
round_id: ndarray of shape (n_sample, ).
json_path: The path used to save results.
Retures:
--------
None
"""
pred = torch.Tensor(pred)
ranks = scores_to_ranks(pred)
ranks_json = []
for i, img_id in enumerate(image_id):
if split == 'test':
ranks_json.append({
'image_id':
int(img_id),
'round_id':
int(round_id[i]),
'ranks': [rank.item() for rank in ranks[i][round_id[i] - 1]]
})
elif split == 'val':
for j in range(10):
ranks_json.append({
"image_id":
int(img_id),
"round_id":
int(j + 1),
"ranks": [rank.item() for rank in ranks[i][j]],
})
json.dump(ranks_json, open(json_path, 'w'))
ndcg = NDCG()
# ================================================================================================
# EVALUATION LOOP
# ================================================================================================
model.eval()
ranks_json = []
for _, batch in enumerate(tqdm(val_dataloader)):
for key in batch:
batch[key] = batch[key].to(device)
with torch.no_grad():
output = model(batch)
ranks = scores_to_ranks(output)
for i in range(len(batch["img_ids"])):
# cast into types explicitly to ensure no errors in schema
# round ids are 1-10, not 0-9
if args.split == "test":
ranks_json.append({
"image_id":
batch["img_ids"][i].item(),
"round_id":
int(batch["num_rounds"][i].item()),
"ranks":
[rank.item() for rank in ranks[i][batch["num_rounds"][i] - 1]]
})
else:
for j in range(batch["num_rounds"][i]):
ranks_json.append({
