def test_top_element_is_predicted(self):
# given
predictions = torch.tensor([[0.3, 0.1, 0.0, 0.2]])
ground_truth_idx = torch.tensor([2])
expected_score = 1.0
# when
actual_score = metric.mrr(predictions, ground_truth_idx)
# then
self.assertEqual(expected_score, actual_score)
def test_multiple_element_batch(self):
# given
predictions = torch.tensor([[0.0, 0.1, 0.3, 0.2], [0.0, 0.1, 0.2, 0.3],
[0.3, 0.1, 0.2, 0.0]])
ground_truth_idx = torch.tensor([[2], [2], [1]])
expected_score = 0.25 + 1 / 3 + 0.5
# when
actual_score = metric.mrr(predictions, ground_truth_idx)
# then
self.assertAlmostEqual(expected_score, actual_score)
def test_missing_ground_truth_idx(self):
# given
predictions = torch.tensor([[0.0, 0.1, 0.2, 0.3], [0.0, 0.1, 0.2,
0.3]])
ground_truth_idx = torch.tensor([[4], [4]])
expected_score = 0.0
# when
actual_score = metric.mrr(predictions, ground_truth_idx)
# then
self.assertEqual(expected_score, actual_score)
def model_score(index, tar, k):
NDCG, hit, mrr = [], [], []
for score, target in zip(index, tar):
# todo:NDCG@k
IDCG = 0
# 理想情况下
IDCG += 1 / math.log(2, 2)
DCG = 0
l = list(score)
NDCG.append(error.ndcg(l, [target]))
hit.append(error.hit(l, [target]))
mrr.append(error.mrr(l, [target]))
return NDCG, hit, mrr
def test(model: torch.nn.Module, data_generator: torch_data.DataLoader, entities_count: int,
device: torch.device, epoch_id: int, metric_suffix: str,
) -> METRICS:
examples_count = 0.0
hits_at_1 = 0.0
hits_at_3 = 0.0
hits_at_10 = 0.0
mrr = 0.0
entity_ids = torch.arange(end=entities_count, device=device).unsqueeze(0)
for head, relation, tail in data_generator:
current_batch_size = head.size()[0]
head, relation, tail = head.to(device), relation.to(device), tail.to(device) # [B]
all_entities = entity_ids.repeat(current_batch_size, 1) # [B, e]
heads = head.reshape(-1, 1).repeat(1, all_entities.size()[1]) # [B, e]
relations = relation.reshape(-1, 1).repeat(1, all_entities.size()[1]) # [B, e]
tails = tail.reshape(-1, 1).repeat(1, all_entities.size()[1]) # [B, e]
# Check all possible tails
triplets = torch.stack((heads, relations, all_entities), dim=2).reshape(-1, 3) # [B, e, 3]->[B*e, 3]
tails_predictions = model.predict(triplets).reshape(current_batch_size, -1) # B*e ->[B, e]
# Check all possible heads
triplets = torch.stack((all_entities, relations, tails), dim=2).reshape(-1, 3)
heads_predictions = model.predict(triplets).reshape(current_batch_size, -1)
# Concat predictions
predictions = torch.cat((tails_predictions, heads_predictions), dim=0) # [2*B, e]
ground_truth_entity_id = torch.cat((tail.reshape(-1, 1), head.reshape(-1, 1))) # [2*B, 1] 2B is tail and head
hits_at_1 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=1)
hits_at_3 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=3)
hits_at_10 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=10)
mrr += metric.mrr(predictions, ground_truth_entity_id) # 对于一个query，若第一个正确答案排在第n位，则MRR得分就是 1/n 。（如果没有正确答案，则得分为0）
examples_count += predictions.size()[0]
# hits_at_1_score = hits_at_1 / examples_count * 100
# hits_at_3_score = hits_at_3 / examples_count * 100
# hits_at_10_score = hits_at_10 / examples_count * 100
# mrr_score = mrr / examples_count * 100
hits_at_1_score = hits_at_1 / examples_count
hits_at_3_score = hits_at_3 / examples_count
hits_at_10_score = hits_at_10 / examples_count
mrr_score = mrr / examples_count
# summary_writer.add_scalar('Metrics/Hits_1/' + metric_suffix, hits_at_1_score, global_step=epoch_id)
# summary_writer.add_scalar('Metrics/Hits_3/' + metric_suffix, hits_at_3_score, global_step=epoch_id)
# summary_writer.add_scalar('Metrics/Hits_10/' + metric_suffix, hits_at_10_score, global_step=epoch_id)
# summary_writer.add_scalar('Metrics/MRR/' + metric_suffix, mrr_score, global_step=epoch_id)
return hits_at_1_score, hits_at_3_score, hits_at_10_score, mrr_score
def test(model: torch.nn.Module, data_generator: torch_data.DataLoader, entities_count: int,
summary_writer: tensorboard.SummaryWriter, device: torch.device, epoch_id: int, metric_suffix: str,
) -> METRICS:
examples_count = 0.0
hits_at_1 = 0.0
hits_at_3 = 0.0
hits_at_10 = 0.0
mrr = 0.0
entity_ids = torch.arange(end=entities_count, device=device).unsqueeze(0)
for head, relation, tail in data_generator:
current_batch_size = head.size()[0]
head, relation, tail = head.to(device), relation.to(device), tail.to(device)
all_entities = entity_ids.repeat(current_batch_size, 1)
heads = head.reshape(-1, 1).repeat(1, all_entities.size()[1])
relations = relation.reshape(-1, 1).repeat(1, all_entities.size()[1])
tails = tail.reshape(-1, 1).repeat(1, all_entities.size()[1])
# Check all possible tails
triplets = torch.stack((heads, relations, all_entities), dim=2).reshape(-1, 3)
tails_predictions = model.predict(triplets).reshape(current_batch_size, -1)
# Check all possible heads
triplets = torch.stack((all_entities, relations, tails), dim=2).reshape(-1, 3)
heads_predictions = model.predict(triplets).reshape(current_batch_size, -1)
# Concat predictions
predictions = torch.cat((tails_predictions, heads_predictions), dim=0)
ground_truth_entity_id = torch.cat((tail.reshape(-1, 1), head.reshape(-1, 1)))
hits_at_1 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=1)
hits_at_3 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=3)
hits_at_10 += metric.hit_at_k(predictions, ground_truth_entity_id, device=device, k=10)
mrr += metric.mrr(predictions, ground_truth_entity_id)
examples_count += predictions.size()[0]
hits_at_1_score = hits_at_1 / examples_count * 100
hits_at_3_score = hits_at_3 / examples_count * 100
hits_at_10_score = hits_at_10 / examples_count * 100
mrr_score = mrr / examples_count * 100
summary_writer.add_scalar('Metrics/Hits_1/' + metric_suffix, hits_at_1_score, global_step=epoch_id)
summary_writer.add_scalar('Metrics/Hits_3/' + metric_suffix, hits_at_3_score, global_step=epoch_id)
summary_writer.add_scalar('Metrics/Hits_10/' + metric_suffix, hits_at_10_score, global_step=epoch_id)
summary_writer.add_scalar('Metrics/MRR/' + metric_suffix, mrr_score, global_step=epoch_id)
return hits_at_1_score, hits_at_3_score, hits_at_10_score, mrr_score
def test(
model: torch.nn.Module,
data_generator: torch_data.DataLoader,
entities_count: int,
summary_writer: tensorboard.SummaryWriter,
device: torch.device,
epoch_id: int,
metric_suffix: str,
) -> METRICS:
examples_count = 0.0
hits_at_1 = 0.0
hits_at_3 = 0.0
hits_at_10 = 0.0
mrr = 0.0
entity_ids = torch.arange(end=entities_count, device=device).unsqueeze(
0
) # Returns a 1-D tensor of size entities_count with values from 0 to entities_count, and then Returns a new tensor with a dimension of size one inserted at the specified position/ basically adding another dimension.
for head, relation, tail in data_generator:
# print(head, relation, tail)
current_batch_size = head.size()[0]
head, relation, tail = head.to(device), relation.to(device), tail.to(
device)
all_entities = entity_ids.repeat(
current_batch_size, 1
) # with torch.repeat(), you can specify the number of repeats for each dimension
heads = head.reshape(-1, 1).repeat(1, all_entities.size()[1])
relations = relation.reshape(-1, 1).repeat(1, all_entities.size()[1])
tails = tail.reshape(-1, 1).repeat(1, all_entities.size()[1])
# Check all possible tails
triplets = torch.stack((heads, relations, all_entities),
dim=2).reshape(-1, 3)
# print(triplets)
tails_predictions = model.predict(triplets).reshape(
current_batch_size, -1)
# Check all possible heads
triplets = torch.stack((all_entities, relations, tails),
dim=2).reshape(-1, 3)
heads_predictions = model.predict(triplets).reshape(
current_batch_size, -1)
# Concat predictions
predictions = torch.cat((tails_predictions, heads_predictions), dim=0)
ground_truth_entity_id = torch.cat(
(tail.reshape(-1, 1), head.reshape(-1, 1)))
# Each prediction is an array of N size, where N is no_of_Entity_in_KB, and there are no_of_samples_in_batch * 2 (head & tail) ground_truth and column level prediction
# https://medium.com/@m_n_malaeb/recall-and-precision-at-k-for-recommender-systems-618483226c54
hits_at_1 += metric.hit_at_k(predictions,
ground_truth_entity_id,
device=device,
k=1)
hits_at_3 += metric.hit_at_k(predictions,
ground_truth_entity_id,
device=device,
k=3)
hits_at_10 += metric.hit_at_k(predictions,
ground_truth_entity_id,
device=device,
k=10)
mrr += metric.mrr(predictions, ground_truth_entity_id)
examples_count += predictions.size()[0]
hits_at_1_score = hits_at_1 / examples_count * 100
hits_at_3_score = hits_at_3 / examples_count * 100
hits_at_10_score = hits_at_10 / examples_count * 100
mrr_score = mrr / examples_count * 100
summary_writer.add_scalar('Metrics/Hits_1/' + metric_suffix,
hits_at_1_score,
global_step=epoch_id)
summary_writer.add_scalar('Metrics/Hits_3/' + metric_suffix,
hits_at_3_score,
global_step=epoch_id)
summary_writer.add_scalar('Metrics/Hits_10/' + metric_suffix,
hits_at_10_score,
global_step=epoch_id)
summary_writer.add_scalar('Metrics/MRR/' + metric_suffix,
mrr_score,
global_step=epoch_id)
return hits_at_1_score, hits_at_3_score, hits_at_10_score, mrr_score
