def test_earl_stopping_with_optimize_epochs(self): """Assert that the pipeline raises a value error.""" with self.assertRaises(ValueError): hpo_pipeline(dataset='kinships', model='transe', stopper='early', training_kwargs_ranges=dict(epochs=...))
def test_fail_invalid_kwarg_ranges(self): """Test that an exception is thrown if an incorrect argument is passed.""" with self.assertRaises(ExtraKeysError) as e: hpo_pipeline( dataset="Nations", model="TransE", n_trials=1, training_loop="sLCWA", training_kwargs=dict(num_epochs=5, use_tqdm=False), negative_sampler_kwargs_ranges=dict(garbage_key=dict( type=int, low=1, high=100), ), ) self.assertEqual(["garbage_key"], e.exception.args[0])
def test_failing_trials(self): """Test whether failing trials are correctly reported.""" class MockResultTracker(MagicMock, ResultTracker): """A mock result tracker.""" tracker_resolver.register(MockResultTracker) mock_result_tracker = MockResultTracker() mock_result_tracker.end_run = MagicMock() result = hpo_pipeline( dataset="nations", model="distmult", model_kwargs_ranges=dict( embedding_dim=dict( type=int, low=-10, high=-1, # will fail ), ), n_trials=1, result_tracker=mock_result_tracker, ) # verify failure assert all(t.state == TrialState.FAIL for t in result.study.trials) assert all(ca[1]["success"] is False for ca in mock_result_tracker.end_run.call_args_list)
def _help_test_hpo(self, **kwargs): hpo_pipeline_result = hpo_pipeline( **kwargs, model="TransE", n_trials=1, training_kwargs=dict(num_epochs=1, use_tqdm=False), evaluation_kwargs=dict(use_tqdm=False), ) with tempfile.TemporaryDirectory() as directory: hpo_pipeline_result.save_to_directory(directory) return hpo_pipeline_result
def test_run(self): """Test simply making a study.""" hpo_pipeline_result = hpo_pipeline( dataset="nations", model="TransE", training_kwargs=dict(num_epochs=5, use_tqdm=False), n_trials=2, ) df = hpo_pipeline_result.study.trials_dataframe(multi_index=True) # Check a model param is optimized self.assertIn(("params", "model.embedding_dim"), df.columns) # Check a loss param is optimized self.assertIn(("params", "loss.margin"), df.columns) self.assertNotIn(("params", "training.num_epochs"), df.columns)
def test_specified_loss_hyperparameter(self): """Test making a study that has a specified loss hyper-parameter.""" hpo_pipeline_result = hpo_pipeline( dataset="nations", model="TransE", loss_kwargs=dict(margin=1.0), training_kwargs=dict(num_epochs=5, use_tqdm=False), n_trials=2, ) df = hpo_pipeline_result.study.trials_dataframe(multi_index=True) # Check a model param is optimized self.assertIn(("params", "model.embedding_dim"), df.columns) # Check a loss param is NOT optimized self.assertNotIn(("params", "loss.margin"), df.columns)
def test_run(self): """Test simply making a study.""" hpo_pipeline_result = hpo_pipeline( dataset=NationsLiteral, model='DistMultLiteral', training_kwargs=dict(num_epochs=5, use_tqdm=False), n_trials=2, ) df = hpo_pipeline_result.study.trials_dataframe(multi_index=True) # Check a model param is optimized self.assertIn(('params', 'model.embedding_dim'), df.columns) # Check a loss param is optimized self.assertIn(('params', 'loss.margin'), df.columns) self.assertNotIn(('params', 'training.num_epochs'), df.columns)
def test_specified_model_hyperparameter(self): """Test making a study that has a specified model hyper-parameter.""" target_embedding_dim = 50 hpo_pipeline_result = hpo_pipeline( dataset='nations', model='TransE', model_kwargs=dict(embedding_dim=target_embedding_dim), training_kwargs=dict(num_epochs=5, use_tqdm=False), n_trials=2, ) df = hpo_pipeline_result.study.trials_dataframe(multi_index=True) # Check a model param is NOT optimized self.assertNotIn(('params', 'model.embedding_dim'), df.columns) # Check a loss param is optimized self.assertIn(('params', 'loss.margin'), df.columns)
def test_custom_tf(self): """Test using a custom triples factories with HPO. .. seealso:: https://github.com/pykeen/pykeen/issues/230 """ tf = TriplesFactory.from_path(path=NATIONS_TRAIN_PATH) training, testing, validation = tf.split([.8, .1, .1], random_state=0) hpo_pipeline_result = hpo_pipeline( training=training, testing=testing, validation=validation, model='TransE', n_trials=2, training_kwargs=dict(num_epochs=2), ) with tempfile.TemporaryDirectory() as directory: hpo_pipeline_result.save_to_directory(directory)
def test_specified_range(self): """Test making a study that has a specified hyper-parameter.""" hpo_pipeline_result = hpo_pipeline( dataset='nations', model='TransE', model_kwargs_ranges=dict( embedding_dim=dict(type=int, low=60, high=80, q=10), ), loss_kwargs_ranges=dict( margin=dict(type=int, low=1, high=2), ), training_kwargs=dict(num_epochs=5, use_tqdm=False), n_trials=2, ) df = hpo_pipeline_result.study.trials_dataframe(multi_index=True) self.assertIn(('params', 'model.embedding_dim'), df.columns) self.assertTrue(df[('params', 'model.embedding_dim')].isin({60., 70., 80.}).all()) self.assertIn(('params', 'loss.margin'), df.columns) self.assertTrue(df[('params', 'loss.margin')].isin({1, 2}).all())