def test_earl_stopping_with_optimize_epochs(self):
"""Assert that the pipeline raises a value error."""
with self.assertRaises(ValueError):
hpo_pipeline(dataset='kinships',
model='transe',
stopper='early',
training_kwargs_ranges=dict(epochs=...))
def test_fail_invalid_kwarg_ranges(self):
"""Test that an exception is thrown if an incorrect argument is passed."""
with self.assertRaises(ExtraKeysError) as e:
hpo_pipeline(
dataset="Nations",
model="TransE",
n_trials=1,
training_loop="sLCWA",
training_kwargs=dict(num_epochs=5, use_tqdm=False),
negative_sampler_kwargs_ranges=dict(garbage_key=dict(
type=int, low=1, high=100), ),
)
self.assertEqual(["garbage_key"], e.exception.args[0])
def test_failing_trials(self):
"""Test whether failing trials are correctly reported."""
class MockResultTracker(MagicMock, ResultTracker):
"""A mock result tracker."""
tracker_resolver.register(MockResultTracker)
mock_result_tracker = MockResultTracker()
mock_result_tracker.end_run = MagicMock()
result = hpo_pipeline(
dataset="nations",
model="distmult",
model_kwargs_ranges=dict(
embedding_dim=dict(
type=int,
low=-10,
high=-1, # will fail
), ),
n_trials=1,
result_tracker=mock_result_tracker,
)
# verify failure
assert all(t.state == TrialState.FAIL for t in result.study.trials)
assert all(ca[1]["success"] is False
for ca in mock_result_tracker.end_run.call_args_list)
def _help_test_hpo(self, **kwargs):
hpo_pipeline_result = hpo_pipeline(
**kwargs,
model="TransE",
n_trials=1,
training_kwargs=dict(num_epochs=1, use_tqdm=False),
evaluation_kwargs=dict(use_tqdm=False),
)
with tempfile.TemporaryDirectory() as directory:
hpo_pipeline_result.save_to_directory(directory)
return hpo_pipeline_result
def test_run(self):
"""Test simply making a study."""
hpo_pipeline_result = hpo_pipeline(
dataset="nations",
model="TransE",
training_kwargs=dict(num_epochs=5, use_tqdm=False),
n_trials=2,
)
df = hpo_pipeline_result.study.trials_dataframe(multi_index=True)
# Check a model param is optimized
self.assertIn(("params", "model.embedding_dim"), df.columns)
# Check a loss param is optimized
self.assertIn(("params", "loss.margin"), df.columns)
self.assertNotIn(("params", "training.num_epochs"), df.columns)
def test_specified_loss_hyperparameter(self):
"""Test making a study that has a specified loss hyper-parameter."""
hpo_pipeline_result = hpo_pipeline(
dataset="nations",
model="TransE",
loss_kwargs=dict(margin=1.0),
training_kwargs=dict(num_epochs=5, use_tqdm=False),
n_trials=2,
)
df = hpo_pipeline_result.study.trials_dataframe(multi_index=True)
# Check a model param is optimized
self.assertIn(("params", "model.embedding_dim"), df.columns)
# Check a loss param is NOT optimized
self.assertNotIn(("params", "loss.margin"), df.columns)
def test_run(self):
"""Test simply making a study."""
hpo_pipeline_result = hpo_pipeline(
dataset=NationsLiteral,
model='DistMultLiteral',
training_kwargs=dict(num_epochs=5, use_tqdm=False),
n_trials=2,
)
df = hpo_pipeline_result.study.trials_dataframe(multi_index=True)
# Check a model param is optimized
self.assertIn(('params', 'model.embedding_dim'), df.columns)
# Check a loss param is optimized
self.assertIn(('params', 'loss.margin'), df.columns)
self.assertNotIn(('params', 'training.num_epochs'), df.columns)
def test_specified_model_hyperparameter(self):
"""Test making a study that has a specified model hyper-parameter."""
target_embedding_dim = 50
hpo_pipeline_result = hpo_pipeline(
dataset='nations',
model='TransE',
model_kwargs=dict(embedding_dim=target_embedding_dim),
training_kwargs=dict(num_epochs=5, use_tqdm=False),
n_trials=2,
)
df = hpo_pipeline_result.study.trials_dataframe(multi_index=True)
# Check a model param is NOT optimized
self.assertNotIn(('params', 'model.embedding_dim'), df.columns)
# Check a loss param is optimized
self.assertIn(('params', 'loss.margin'), df.columns)
def test_custom_tf(self):
"""Test using a custom triples factories with HPO.
.. seealso:: https://github.com/pykeen/pykeen/issues/230
"""
tf = TriplesFactory.from_path(path=NATIONS_TRAIN_PATH)
training, testing, validation = tf.split([.8, .1, .1], random_state=0)
hpo_pipeline_result = hpo_pipeline(
training=training,
testing=testing,
validation=validation,
model='TransE',
n_trials=2,
training_kwargs=dict(num_epochs=2),
)
with tempfile.TemporaryDirectory() as directory:
hpo_pipeline_result.save_to_directory(directory)
def test_specified_range(self):
"""Test making a study that has a specified hyper-parameter."""
hpo_pipeline_result = hpo_pipeline(
dataset='nations',
model='TransE',
model_kwargs_ranges=dict(
embedding_dim=dict(type=int, low=60, high=80, q=10),
),
loss_kwargs_ranges=dict(
margin=dict(type=int, low=1, high=2),
),
training_kwargs=dict(num_epochs=5, use_tqdm=False),
n_trials=2,
)
df = hpo_pipeline_result.study.trials_dataframe(multi_index=True)
self.assertIn(('params', 'model.embedding_dim'), df.columns)
self.assertTrue(df[('params', 'model.embedding_dim')].isin({60., 70., 80.}).all())
self.assertIn(('params', 'loss.margin'), df.columns)
self.assertTrue(df[('params', 'loss.margin')].isin({1, 2}).all())
