class TestEvaluationFiltering(unittest.TestCase):
"""Tests for testing the correct filtering of positive triples of the evaluation procedure."""
def setUp(self):
"""Prepare for testing the evaluation filtering."""
self.evaluator = RankBasedEvaluator(
filtered=True, automatic_memory_optimization=False)
self.triples_factory = Nations().training
self.model = FixedModel(triples_factory=self.triples_factory)
# The MockModel gives the highest score to the highest entity id
max_score = self.triples_factory.num_entities - 1
# The test triples are created to yield the third highest score on both head and tail prediction
self.test_triples = torch.tensor([[max_score - 2, 0, max_score - 2]])
# Write new mapped triples to the model, since the model's triples will be used to filter
# These triples are created to yield the highest score on both head and tail prediction for the
# test triple at hand
self.training_triples = torch.tensor([
[max_score - 2, 0, max_score],
[max_score, 0, max_score - 2],
], )
# The validation triples are created to yield the second highest score on both head and tail prediction for the
# test triple at hand
self.validation_triples = torch.tensor([
[max_score - 2, 0, max_score - 1],
[max_score - 1, 0, max_score - 2],
], )
def test_evaluation_filtering_without_validation_triples(self):
"""Test if the evaluator's triple filtering works as expected."""
eval_results = self.evaluator.evaluate(
model=self.model,
mapped_triples=self.test_triples,
additional_filter_triples=self.training_triples,
batch_size=1,
use_tqdm=False,
)
assert eval_results.get_metric(
name="mr") == 2, "The mean rank should equal 2"
def test_evaluation_filtering_with_validation_triples(self):
"""Test if the evaluator's triple filtering works as expected when including additional filter triples."""
eval_results = self.evaluator.evaluate(
model=self.model,
mapped_triples=self.test_triples,
additional_filter_triples=[
self.training_triples,
self.validation_triples,
],
batch_size=1,
use_tqdm=False,
)
assert eval_results.get_metric(
name="mr") == 1, "The rank should equal 1"
[1, 2, 1], [1, 2, 1], [1, 2, 1], [1, 2, 1], [1, 2, 1],
[1, 2, 1], [1, 2, 1], [1, 2, 1], [1, 2, 1], [1, 2, 1],
[1, 2, 1], [1, 2, 1], [1, 2, 1], [1, 2, 1], [1, 2, 1],
[1, 2, 1]])
entity_initializer = lambda t: torch.as_tensor(np_array, dtype=torch.float32)
model = RGCN(triples_factory=training,
entity_representations=entity_initializer)
optimizer = Adam(params=model.get_grad_params())
training_loop = SLCWATrainingLoop(
model=model,
triples_factory=training,
optimizer=optimizer,
)
# train
training_loop.train(
triples_factory=training,
num_epochs=500,
batch_size=256,
)
# evaluate
evaluator = RankBasedEvaluator(ks=[50])
mapped_triples = testing.mapped_triples
foo = evaluator.evaluate(model=model,
mapped_triples=mapped_triples,
batch_size=1024)