def _run_GS_for_N_rounds(self, gs: GenerationStrategy, exp: Experiment, num_rounds: int) -> List[int]: could_gen = [] for _ in range(num_rounds): ( num_trials_to_gen, opt_complete, ) = gs.current_generator_run_limit() self.assertFalse(opt_complete) could_gen.append(num_trials_to_gen) trials = [] for _ in range(num_trials_to_gen): gr = gs.gen( experiment=exp, pending_observations=get_pending(experiment=exp), ) trials.append( exp.new_trial(gr).mark_running(no_runner_required=True)) for trial in trials: exp.attach_data(get_branin_data(trial_indices=[trial.index])) trial.mark_completed() return could_gen
def test_current_generator_run_limit(self): NUM_INIT_TRIALS = 5 SECOND_STEP_PARALLELISM = 3 NUM_ROUNDS = 4 exp = get_branin_experiment() sobol_gs_with_parallelism_limits = GenerationStrategy(steps=[ GenerationStep( model=Models.SOBOL, num_trials=NUM_INIT_TRIALS, min_trials_observed=3, ), GenerationStep( model=Models.SOBOL, num_trials=-1, max_parallelism=SECOND_STEP_PARALLELISM, ), ]) sobol_gs_with_parallelism_limits._experiment = exp could_gen = [] for _ in range(NUM_ROUNDS): ( num_trials_to_gen, opt_complete, ) = sobol_gs_with_parallelism_limits.current_generator_run_limit() self.assertFalse(opt_complete) could_gen.append(num_trials_to_gen) trials = [] for _ in range(num_trials_to_gen): gr = sobol_gs_with_parallelism_limits.gen( experiment=exp, pending_observations=get_pending(experiment=exp), ) trials.append( exp.new_trial(gr).mark_running(no_runner_required=True)) for trial in trials: exp.attach_data(get_branin_data(trial_indices=[trial.index])) trial.mark_completed() # We expect trials from first generation step + trials from remaining rounds in # batches limited by parallelism setting in the second step. self.assertEqual( len(exp.trials), NUM_INIT_TRIALS + (NUM_ROUNDS - 1) * SECOND_STEP_PARALLELISM, ) self.assertTrue(all(t.status.is_completed for t in exp.trials.values())) self.assertEqual(could_gen, [NUM_INIT_TRIALS] + [SECOND_STEP_PARALLELISM] * (NUM_ROUNDS - 1))
def test_current_generator_run_limit(self): NUM_INIT_TRIALS = 5 SECOND_STEP_PARALLELISM = 3 NUM_ROUNDS = 4 exp = get_branin_experiment() sobol_gs_with_parallelism_limits = GenerationStrategy(steps=[ GenerationStep( model=Models.SOBOL, num_trials=NUM_INIT_TRIALS, min_trials_observed=3, ), GenerationStep( model=Models.SOBOL, num_trials=(NUM_ROUNDS - 1) * SECOND_STEP_PARALLELISM, max_parallelism=SECOND_STEP_PARALLELISM, ), ]) sobol_gs_with_parallelism_limits._experiment = exp could_gen = self._run_GS_for_N_rounds( gs=sobol_gs_with_parallelism_limits, exp=exp, num_rounds=NUM_ROUNDS) # Optimization should now be complete. ( num_trials_to_gen, opt_complete, ) = sobol_gs_with_parallelism_limits.current_generator_run_limit() self.assertTrue(opt_complete) self.assertEqual(num_trials_to_gen, 0) # We expect trials from first generation step + trials from remaining rounds in # batches limited by parallelism setting in the second step. self.assertEqual( len(exp.trials), NUM_INIT_TRIALS + (NUM_ROUNDS - 1) * SECOND_STEP_PARALLELISM, ) self.assertTrue(all(t.status.is_completed for t in exp.trials.values())) self.assertEqual(could_gen, [NUM_INIT_TRIALS] + [SECOND_STEP_PARALLELISM] * (NUM_ROUNDS - 1))