def test_run_preattached_trials_only(self):
# assert that pre-attached trials run when max_trials = number of
# pre-attached trials
scheduler = Scheduler(
experiment=self.branin_experiment, # Has runner and metrics.
generation_strategy=self.two_sobol_steps_GS,
options=SchedulerOptions(
init_seconds_between_polls=
0.1, # Short between polls so test is fast.
),
)
trial = scheduler.experiment.new_trial()
parameter_dict = {"x1": 5, "x2": 5}
trial.add_arm(Arm(parameters=parameter_dict))
with self.assertRaisesRegex(
UserInputError,
"number of pre-attached candidate trials .* is greater than"):
scheduler.run_n_trials(max_trials=0)
scheduler.run_n_trials(max_trials=1)
self.assertEqual(len(scheduler.experiment.trials), 1)
self.assertDictEqual(scheduler.experiment.trials[0].arm.parameters,
parameter_dict)
self.assertTrue( # Make sure all trials got to complete.
all(t.completed_successfully
for t in scheduler.experiment.trials.values()))
def test_get_best_trial(self):
scheduler = Scheduler(
experiment=self.branin_experiment, # Has runner and metrics.
generation_strategy=self.two_sobol_steps_GS,
options=SchedulerOptions(
init_seconds_between_polls=
0.1, # Short between polls so test is fast.
),
)
self.assertIsNone(scheduler.get_best_parameters())
scheduler.run_n_trials(max_trials=1)
trial, params, _arm = scheduler.get_best_trial()
just_params, _just_arm = scheduler.get_best_parameters()
just_params_unmodeled, _just_arm_unmodled = scheduler.get_best_parameters(
use_model_predictions=False)
with self.assertRaisesRegex(NotImplementedError,
"Please use `get_best_parameters`"):
scheduler.get_pareto_optimal_parameters()
self.assertEqual(trial, 0)
self.assertIn("x1", params)
self.assertIn("x2", params)
self.assertEqual(params, just_params)
self.assertEqual(params, just_params_unmodeled)
def test_optimization_complete(self, _):
# With runners & metrics, `Scheduler.run_all_trials` should run.
scheduler = Scheduler(
experiment=self.branin_experiment, # Has runner and metrics.
generation_strategy=self.two_sobol_steps_GS,
options=SchedulerOptions(
max_pending_trials=100,
init_seconds_between_polls=
0.1, # Short between polls so test is fast.
),
)
scheduler.run_n_trials(max_trials=1)
# no trials should run if _gen_multiple throws an OptimizationComplete error
self.assertEqual(len(scheduler.experiment.trials), 0)
def test_stop_trial(self):
# With runners & metrics, `Scheduler.run_all_trials` should run.
scheduler = Scheduler(
experiment=self.branin_experiment, # Has runner and metrics.
generation_strategy=self.two_sobol_steps_GS,
options=SchedulerOptions(
init_seconds_between_polls=
0.1, # Short between polls so test is fast.
),
)
with patch.object(scheduler.experiment.runner,
"stop",
return_value=None) as mock_runner_stop:
scheduler.run_n_trials(max_trials=1)
scheduler.stop_trial_runs(trials=[scheduler.experiment.trials[0]])
mock_runner_stop.assert_called_once()
def test_base_report_results(self):
self.branin_experiment.runner = NoReportResultsRunner()
scheduler = Scheduler(
experiment=self.branin_experiment, # Has runner and metrics.
generation_strategy=self.two_sobol_steps_GS,
options=SchedulerOptions(init_seconds_between_polls=0, ),
)
self.assertEqual(scheduler.run_n_trials(max_trials=3),
OptimizationResult())
def test_run_n_trials(self):
# With runners & metrics, `Scheduler.run_all_trials` should run.
scheduler = Scheduler(
experiment=self.branin_experiment, # Has runner and metrics.
generation_strategy=self.two_sobol_steps_GS,
options=SchedulerOptions(
init_seconds_between_polls=
0.1, # Short between polls so test is fast.
),
)
scheduler.run_n_trials(max_trials=1)
self.assertEqual(len(scheduler.experiment.trials), 1)
scheduler.run_n_trials(max_trials=10)
self.assertTrue( # Make sure all trials got to complete.
all(t.completed_successfully
for t in scheduler.experiment.trials.values()))
# Check that all the data, fetched during optimization, was attached to the
# experiment.
dat = scheduler.experiment.fetch_data().df
self.assertEqual(set(dat["trial_index"].values), set(range(11)))
def test_get_best_trial_moo(self):
experiment = get_branin_experiment_with_multi_objective()
experiment.runner = self.runner
scheduler = Scheduler(
experiment=experiment,
generation_strategy=self.sobol_GPEI_GS,
options=SchedulerOptions(init_seconds_between_polls=0.1),
)
scheduler.run_n_trials(max_trials=1)
with self.assertRaisesRegex(
NotImplementedError,
"Please use `get_pareto_optimal_parameters`"):
scheduler.get_best_trial()
with self.assertRaisesRegex(
NotImplementedError,
"Please use `get_pareto_optimal_parameters`"):
scheduler.get_best_parameters()
self.assertIsNotNone(scheduler.get_pareto_optimal_parameters())