def test_top_element_is_predicted(self): # given predictions = torch.tensor([[0.3, 0.1, 0.0, 0.2]]) ground_truth_idx = torch.tensor([2]) expected_score = 1.0 # when actual_score = metric.mrr(predictions, ground_truth_idx) # then self.assertEqual(expected_score, actual_score)
def test_multiple_element_batch(self): # given predictions = torch.tensor([[0.0, 0.1, 0.3, 0.2], [0.0, 0.1, 0.2, 0.3], [0.3, 0.1, 0.2, 0.0]]) ground_truth_idx = torch.tensor([[2], [2], [1]]) expected_score = 0.25 + 1 / 3 + 0.5 # when actual_score = metric.mrr(predictions, ground_truth_idx) # then self.assertAlmostEqual(expected_score, actual_score)
def test_missing_ground_truth_idx(self): # given predictions = torch.tensor([[0.0, 0.1, 0.2, 0.3], [0.0, 0.1, 0.2, 0.3]]) ground_truth_idx = torch.tensor([[4], [4]]) expected_score = 0.0 # when actual_score = metric.mrr(predictions, ground_truth_idx) # then self.assertEqual(expected_score, actual_score)
def model_score(index, tar, k): NDCG, hit, mrr = [], [], [] for score, target in zip(index, tar): # todo:NDCG@k IDCG = 0 # 理想情况下 IDCG += 1 / math.log(2, 2) DCG = 0 l = list(score) NDCG.append(error.ndcg(l, [target])) hit.append(error.hit(l, [target])) mrr.append(error.mrr(l, [target])) return NDCG, hit, mrr
def test(model: torch.nn.Module, data_generator: torch_data.DataLoader, entities_count: int, device: torch.device, epoch_id: int, metric_suffix: str, ) -> METRICS: examples_count = 0.0 hits_at_1 = 0.0 hits_at_3 = 0.0 hits_at_10 = 0.0 mrr = 0.0 entity_ids = torch.arange(end=entities_count, device=device).unsqueeze(0) for head, relation, tail in data_generator: current_batch_size = head.size()[0] head, relation, tail = head.to(device), relation.to(device), tail.to(device) # [B] all_entities = entity_ids.repeat(current_batch_size, 1) # [B, e] heads = head.reshape(-1, 1).repeat(1, all_entities.size()[1]) # [B, e] relations = relation.reshape(-1, 1).repeat(1, all_entities.size()[1]) # [B, e] tails = tail.reshape(-1, 1).repeat(1, all_entities.size()[1]) # [B, e] # Check all possible tails triplets = torch.stack((heads, relations, all_entities), dim=2).reshape(-1, 3) # [B, e, 3]->[B*e, 3] tails_predictions = model.predict(triplets).reshape(current_batch_size, -1) # B*e ->[B, e] # Check all possible heads triplets = torch.stack((all_entities, relations, tails), dim=2).reshape(-1, 3) heads_predictions = model.predict(triplets).reshape(current_batch_size, -1) # Concat predictions predictions = torch.cat((tails_predictions, heads_predictions), dim=0) # [2*B, e] ground_truth_entity_id = torch.cat((tail.reshape(-1, 1), head.reshape(-1, 1))) # [2*B, 1] 2B is tail and head hits_at_1 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=1) hits_at_3 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=3) hits_at_10 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=10) mrr += metric.mrr(predictions, ground_truth_entity_id) # 对于一个query,若第一个正确答案排在第n位,则MRR得分就是 1/n 。(如果没有正确答案,则得分为0) examples_count += predictions.size()[0] # hits_at_1_score = hits_at_1 / examples_count * 100 # hits_at_3_score = hits_at_3 / examples_count * 100 # hits_at_10_score = hits_at_10 / examples_count * 100 # mrr_score = mrr / examples_count * 100 hits_at_1_score = hits_at_1 / examples_count hits_at_3_score = hits_at_3 / examples_count hits_at_10_score = hits_at_10 / examples_count mrr_score = mrr / examples_count # summary_writer.add_scalar('Metrics/Hits_1/' + metric_suffix, hits_at_1_score, global_step=epoch_id) # summary_writer.add_scalar('Metrics/Hits_3/' + metric_suffix, hits_at_3_score, global_step=epoch_id) # summary_writer.add_scalar('Metrics/Hits_10/' + metric_suffix, hits_at_10_score, global_step=epoch_id) # summary_writer.add_scalar('Metrics/MRR/' + metric_suffix, mrr_score, global_step=epoch_id) return hits_at_1_score, hits_at_3_score, hits_at_10_score, mrr_score
def test(model: torch.nn.Module, data_generator: torch_data.DataLoader, entities_count: int, summary_writer: tensorboard.SummaryWriter, device: torch.device, epoch_id: int, metric_suffix: str, ) -> METRICS: examples_count = 0.0 hits_at_1 = 0.0 hits_at_3 = 0.0 hits_at_10 = 0.0 mrr = 0.0 entity_ids = torch.arange(end=entities_count, device=device).unsqueeze(0) for head, relation, tail in data_generator: current_batch_size = head.size()[0] head, relation, tail = head.to(device), relation.to(device), tail.to(device) all_entities = entity_ids.repeat(current_batch_size, 1) heads = head.reshape(-1, 1).repeat(1, all_entities.size()[1]) relations = relation.reshape(-1, 1).repeat(1, all_entities.size()[1]) tails = tail.reshape(-1, 1).repeat(1, all_entities.size()[1]) # Check all possible tails triplets = torch.stack((heads, relations, all_entities), dim=2).reshape(-1, 3) tails_predictions = model.predict(triplets).reshape(current_batch_size, -1) # Check all possible heads triplets = torch.stack((all_entities, relations, tails), dim=2).reshape(-1, 3) heads_predictions = model.predict(triplets).reshape(current_batch_size, -1) # Concat predictions predictions = torch.cat((tails_predictions, heads_predictions), dim=0) ground_truth_entity_id = torch.cat((tail.reshape(-1, 1), head.reshape(-1, 1))) hits_at_1 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=1) hits_at_3 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=3) hits_at_10 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=10) mrr += metric.mrr(predictions, ground_truth_entity_id) examples_count += predictions.size()[0] hits_at_1_score = hits_at_1 / examples_count * 100 hits_at_3_score = hits_at_3 / examples_count * 100 hits_at_10_score = hits_at_10 / examples_count * 100 mrr_score = mrr / examples_count * 100 summary_writer.add_scalar('Metrics/Hits_1/' + metric_suffix, hits_at_1_score, global_step=epoch_id) summary_writer.add_scalar('Metrics/Hits_3/' + metric_suffix, hits_at_3_score, global_step=epoch_id) summary_writer.add_scalar('Metrics/Hits_10/' + metric_suffix, hits_at_10_score, global_step=epoch_id) summary_writer.add_scalar('Metrics/MRR/' + metric_suffix, mrr_score, global_step=epoch_id) return hits_at_1_score, hits_at_3_score, hits_at_10_score, mrr_score
def test( model: torch.nn.Module, data_generator: torch_data.DataLoader, entities_count: int, summary_writer: tensorboard.SummaryWriter, device: torch.device, epoch_id: int, metric_suffix: str, ) -> METRICS: examples_count = 0.0 hits_at_1 = 0.0 hits_at_3 = 0.0 hits_at_10 = 0.0 mrr = 0.0 entity_ids = torch.arange(end=entities_count, device=device).unsqueeze( 0 ) # Returns a 1-D tensor of size entities_count with values from 0 to entities_count, and then Returns a new tensor with a dimension of size one inserted at the specified position/ basically adding another dimension. for head, relation, tail in data_generator: # print(head, relation, tail) current_batch_size = head.size()[0] head, relation, tail = head.to(device), relation.to(device), tail.to( device) all_entities = entity_ids.repeat( current_batch_size, 1 ) # with torch.repeat(), you can specify the number of repeats for each dimension heads = head.reshape(-1, 1).repeat(1, all_entities.size()[1]) relations = relation.reshape(-1, 1).repeat(1, all_entities.size()[1]) tails = tail.reshape(-1, 1).repeat(1, all_entities.size()[1]) # Check all possible tails triplets = torch.stack((heads, relations, all_entities), dim=2).reshape(-1, 3) # print(triplets) tails_predictions = model.predict(triplets).reshape( current_batch_size, -1) # Check all possible heads triplets = torch.stack((all_entities, relations, tails), dim=2).reshape(-1, 3) heads_predictions = model.predict(triplets).reshape( current_batch_size, -1) # Concat predictions predictions = torch.cat((tails_predictions, heads_predictions), dim=0) ground_truth_entity_id = torch.cat( (tail.reshape(-1, 1), head.reshape(-1, 1))) # Each prediction is an array of N size, where N is no_of_Entity_in_KB, and there are no_of_samples_in_batch * 2 (head & tail) ground_truth and column level prediction # https://medium.com/@m_n_malaeb/recall-and-precision-at-k-for-recommender-systems-618483226c54 hits_at_1 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=1) hits_at_3 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=3) hits_at_10 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=10) mrr += metric.mrr(predictions, ground_truth_entity_id) examples_count += predictions.size()[0] hits_at_1_score = hits_at_1 / examples_count * 100 hits_at_3_score = hits_at_3 / examples_count * 100 hits_at_10_score = hits_at_10 / examples_count * 100 mrr_score = mrr / examples_count * 100 summary_writer.add_scalar('Metrics/Hits_1/' + metric_suffix, hits_at_1_score, global_step=epoch_id) summary_writer.add_scalar('Metrics/Hits_3/' + metric_suffix, hits_at_3_score, global_step=epoch_id) summary_writer.add_scalar('Metrics/Hits_10/' + metric_suffix, hits_at_10_score, global_step=epoch_id) summary_writer.add_scalar('Metrics/MRR/' + metric_suffix, mrr_score, global_step=epoch_id) return hits_at_1_score, hits_at_3_score, hits_at_10_score, mrr_score