def setUp(self):
"""Prepare for testing the evaluation filtering."""
self.evaluator = RankBasedEvaluator(
filtered=True, automatic_memory_optimization=False)
self.triples_factory = Nations().training
self.model = FixedModel(triples_factory=self.triples_factory)
# The MockModel gives the highest score to the highest entity id
max_score = self.triples_factory.num_entities - 1
# The test triples are created to yield the third highest score on both head and tail prediction
self.test_triples = torch.tensor([[max_score - 2, 0, max_score - 2]])
# Write new mapped triples to the model, since the model's triples will be used to filter
# These triples are created to yield the highest score on both head and tail prediction for the
# test triple at hand
self.training_triples = torch.tensor([
[max_score - 2, 0, max_score],
[max_score, 0, max_score - 2],
], )
# The validation triples are created to yield the second highest score on both head and tail prediction for the
# test triple at hand
self.validation_triples = torch.tensor([
[max_score - 2, 0, max_score - 1],
[max_score - 1, 0, max_score - 2],
], )
def setUpClass(cls):
"""Set up a shared result."""
cls.device = resolve_device("cuda")
cls.dataset = Nations()
cls.model = FixedModel(triples_factory=cls.dataset.training)
# The MockModel gives the highest score to the highest entity id
max_score = cls.dataset.num_entities - 1
# The test triples are created to yield the third highest score on both head and tail prediction
cls.dataset.testing.mapped_triples = torch.tensor(
[[max_score - 2, 0, max_score - 2]])
# Write new mapped triples to the model, since the model's triples will be used to filter
# These triples are created to yield the highest score on both head and tail prediction for the
# test triple at hand
cls.dataset.training.mapped_triples = torch.tensor([
[max_score - 2, 0, max_score],
[max_score, 0, max_score - 2],
], )
# The validation triples are created to yield the second highest score on both head and tail prediction for the
# test triple at hand
cls.dataset.validation.mapped_triples = torch.tensor([
[max_score - 2, 0, max_score - 1],
[max_score - 1, 0, max_score - 2],
], )
def setUp(self):
"""Prepare for testing the evaluation structure."""
self.counter = 1337
self.evaluator = DummyEvaluator(counter=self.counter,
filtered=True,
automatic_memory_optimization=False)
self.dataset = Nations()
self.model = FixedModel(triples_factory=self.dataset.training)
def setUp(self):
"""Prepare for testing the early stopper."""
# Set automatic_memory_optimization to false for tests
self.mock_evaluator = MockEvaluator(self.mock_losses, automatic_memory_optimization=False)
self.triples_factory = Nations()
self.model = FixedModel(triples_factory=self.triples_factory.training)
self.stopper = EarlyStopper(
model=self.model,
evaluator=self.mock_evaluator,
training_triples_factory=self.triples_factory.training,
evaluation_triples_factory=self.triples_factory.validation,
patience=self.patience,
relative_delta=self.delta,
larger_is_better=False,
frequency=1,
)
def setUp(self):
"""Prepare for testing the early stopper."""
# Set automatic_memory_optimization to false for tests
self.mock_evaluator = MockEvaluator(
key=("hits_at_10", SIDE_BOTH, RANK_REALISTIC),
values=self.mock_losses,
automatic_memory_optimization=False,
)
nations = Nations()
self.model = FixedModel(triples_factory=nations.training)
self.stopper = EarlyStopper(
model=self.model,
evaluator=self.mock_evaluator,
training_triples_factory=nations.training,
evaluation_triples_factory=nations.validation,
patience=self.patience,
relative_delta=self.delta,
larger_is_better=False,
)