def setUp(self): self.branin_experiment = get_branin_experiment() self.branin_timestamp_map_metric_experiment = ( get_branin_experiment_with_timestamp_map_metric()) self.runner = SyntheticRunnerWithStatusPolling() self.branin_experiment.runner = self.runner self.branin_experiment._properties[ Keys.IMMUTABLE_SEARCH_SPACE_AND_OPT_CONF] = True self.branin_experiment_no_impl_runner_or_metrics = Experiment( search_space=get_branin_search_space(), optimization_config=OptimizationConfig(objective=Objective( metric=Metric(name="branin"))), ) self.sobol_GPEI_GS = choose_generation_strategy( search_space=get_branin_search_space()) self.two_sobol_steps_GS = GenerationStrategy( # Contrived GS to ensure steps=[ # that `DataRequiredError` is property handled in scheduler. GenerationStep( # This error is raised when not enough trials model=Models. SOBOL, # have been observed to proceed to next num_trials=5, # geneneration step. min_trials_observed=3, max_parallelism=2, ), GenerationStep(model=Models.SOBOL, num_trials=-1, max_parallelism=3), ]) # GS to force the scheduler to poll completed trials after each ran trial. self.sobol_GS_no_parallelism = GenerationStrategy(steps=[ GenerationStep( model=Models.SOBOL, num_trials=-1, max_parallelism=1) ])
def test_percentile_early_stopping_strategy(self): exp = get_branin_experiment_with_timestamp_map_metric(rate=0.5) for i in range(5): trial = exp.new_trial().add_arm( arm=get_branin_arms(n=1, seed=i)[0]) trial.run() exp.attach_data(data=exp.fetch_data()) """ Data looks like this: arm_name metric_name mean sem trial_index timestamp 0 0_0 branin 146.138620 0.0 0 0 1 0_0 branin 117.388086 0.0 0 1 2 0_0 branin 99.950007 0.0 0 2 3 1_0 branin 113.057480 0.0 1 0 4 1_0 branin 90.815154 0.0 1 1 5 1_0 branin 77.324501 0.0 1 2 6 2_0 branin 44.627226 0.0 2 0 7 2_0 branin 35.847504 0.0 2 1 8 2_0 branin 30.522333 0.0 2 2 9 3_0 branin 143.375669 0.0 3 0 10 3_0 branin 115.168704 0.0 3 1 11 3_0 branin 98.060315 0.0 3 2 12 4_0 branin 65.033535 0.0 4 0 13 4_0 branin 52.239184 0.0 4 1 14 4_0 branin 44.479018 0.0 4 2 Looking at the most recent fidelity only (timestamp==2), we have the following metric values for each trial: 0: 99.950007 <-- worst 3: 98.060315 1: 77.324501 4: 44.479018 2: 30.522333 <-- best """ idcs = set(exp.trials.keys()) early_stopping_strategy = PercentileEarlyStoppingStrategy( percentile_threshold=25, ) should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices=idcs, experiment=exp) self.assertEqual(set(should_stop), {0}) early_stopping_strategy = PercentileEarlyStoppingStrategy( percentile_threshold=50, ) should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices=idcs, experiment=exp) self.assertEqual(set(should_stop), {0, 3}) # respect trial_indices argument should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices={0}, experiment=exp) self.assertEqual(set(should_stop), {0}) early_stopping_strategy = PercentileEarlyStoppingStrategy( percentile_threshold=75, ) should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices=idcs, experiment=exp) self.assertEqual(set(should_stop), {0, 3, 1})
def _setupBraninExperiment(self, n: int, incremental: bool = False) -> Experiment: exp = get_branin_experiment_with_timestamp_map_metric(incremental=incremental) batch = exp.new_batch_trial() batch.add_arms_and_weights(arms=get_branin_arms(n=n, seed=0)) batch.run() batch_2 = exp.new_batch_trial() batch_2.add_arms_and_weights(arms=get_branin_arms(n=3 * n, seed=1)) batch_2.run() return exp
def test_threshold_early_stopping_strategy(self): exp = get_branin_experiment_with_timestamp_map_metric(rate=0.5) for i in range(5): trial = exp.new_trial().add_arm( arm=get_branin_arms(n=1, seed=i)[0]) trial.run() exp.attach_data(data=exp.fetch_data()) """ Data looks like this: arm_name metric_name mean sem trial_index timestamp 0 0_0 branin 146.138620 0.0 0 0 1 0_0 branin 117.388086 0.0 0 1 2 0_0 branin 99.950007 0.0 0 2 3 1_0 branin 113.057480 0.0 1 0 4 1_0 branin 90.815154 0.0 1 1 5 1_0 branin 77.324501 0.0 1 2 6 2_0 branin 44.627226 0.0 2 0 7 2_0 branin 35.847504 0.0 2 1 8 2_0 branin 30.522333 0.0 2 2 9 3_0 branin 143.375669 0.0 3 0 10 3_0 branin 115.168704 0.0 3 1 11 3_0 branin 98.060315 0.0 3 2 12 4_0 branin 65.033535 0.0 4 0 13 4_0 branin 52.239184 0.0 4 1 14 4_0 branin 44.479018 0.0 4 2 """ idcs = set(exp.trials.keys()) early_stopping_strategy = ThresholdEarlyStoppingStrategy( metric_threshold=50, min_progression=1) should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices=idcs, experiment=exp) self.assertEqual(set(should_stop), {0, 1, 3}) # respect trial_indices argument should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices={0}, experiment=exp) self.assertEqual(set(should_stop), {0}) # test ignore trial indices early_stopping_strategy = ThresholdEarlyStoppingStrategy( metric_threshold=50, min_progression=1, trial_indices_to_ignore={0}) should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices=idcs, experiment=exp) self.assertEqual(set(should_stop), {1, 3}) # test did not reach min progression early_stopping_strategy = ThresholdEarlyStoppingStrategy( metric_threshold=50, min_progression=3) should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices=idcs, experiment=exp) self.assertEqual(should_stop, {})
def test_percentile_early_stopping_strategy_validation(self): exp = get_branin_experiment() for i in range(5): trial = exp.new_trial().add_arm( arm=get_branin_arms(n=1, seed=i)[0]) trial.run() trial.mark_as(status=TrialStatus.COMPLETED) early_stopping_strategy = PercentileEarlyStoppingStrategy() idcs = set(exp.trials.keys()) exp.attach_data(data=exp.fetch_data()) # Non-MapData attached should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices=idcs, experiment=exp) self.assertEqual(should_stop, {}) exp = get_branin_experiment_with_timestamp_map_metric(rate=0.5) for i in range(5): trial = exp.new_trial().add_arm( arm=get_branin_arms(n=1, seed=i)[0]) trial.run() # No data attached should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices=idcs, experiment=exp) self.assertEqual(should_stop, {}) exp.attach_data(data=exp.fetch_data()) # Not enough learning curves early_stopping_strategy = PercentileEarlyStoppingStrategy( min_curves=6, ) should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices=idcs, experiment=exp) self.assertEqual(should_stop, {}) # Most recent progression below minimum early_stopping_strategy = PercentileEarlyStoppingStrategy( min_progression=3, ) should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices=idcs, experiment=exp) self.assertEqual(should_stop, {}) # True objective metric name self.assertIsNone( early_stopping_strategy.true_objective_metric_name) # default none early_stopping_strategy.true_objective_metric_name = "true_obj_metric" self.assertEqual(early_stopping_strategy.true_objective_metric_name, "true_obj_metric")
def test_percentile_early_stopping_strategy_validation(self): exp = get_branin_experiment() for i in range(5): trial = exp.new_trial().add_arm( arm=get_branin_arms(n=1, seed=i)[0]) trial.run() early_stopping_strategy = PercentileEarlyStoppingStrategy() idcs = set(exp.trials.keys()) exp.attach_data(data=exp.fetch_data()) # Non-MapData attached with self.assertRaisesRegex(ValueError, "expects MapData"): early_stopping_strategy.should_stop_trials_early( trial_indices=idcs, experiment=exp) exp = get_branin_experiment_with_timestamp_map_metric(rate=0.5) for i in range(5): trial = exp.new_trial().add_arm( arm=get_branin_arms(n=1, seed=i)[0]) trial.run() # No data attached should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices=idcs, experiment=exp) self.assertEqual(should_stop, {}) exp.attach_data(data=exp.fetch_data()) # Not enough learning curves early_stopping_strategy = PercentileEarlyStoppingStrategy( min_curves=6, ) should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices=idcs, experiment=exp) self.assertEqual(should_stop, {}) # Most recent progression below minimum early_stopping_strategy = PercentileEarlyStoppingStrategy( min_progression=3, ) should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices=idcs, experiment=exp) self.assertEqual(should_stop, {})
def test_early_stopping_with_unaligned_results(self): # test case 1 exp = get_branin_experiment_with_timestamp_map_metric(rate=0.5) for i in range(5): trial = exp.new_trial().add_arm( arm=get_branin_arms(n=1, seed=i)[0]) trial.run() trial.mark_as(status=TrialStatus.COMPLETED) # manually "unalign" timestamps to simulate real-world scenario # where each curve reports results at different steps data = exp.fetch_data() unaligned_timestamps = [0, 1, 4, 1, 2, 3, 1, 3, 4, 0, 1, 2, 0, 2, 4] data.df.loc[data.df["metric_name"] == "branin", "timestamp"] = unaligned_timestamps exp.attach_data(data=data) """ Dataframe after interpolation: 0 1 2 3 4 timestamp 0 146.138620 NaN NaN 143.375669 65.033535 1 117.388086 113.057480 44.627226 115.168704 58.636359 2 111.575393 90.815154 40.237365 98.060315 52.239184 3 105.762700 77.324501 35.847504 NaN 48.359101 4 99.950007 NaN 30.522333 NaN 44.479018 """ # We consider trials 0, 2, and 4 for early stopping at progression 4, # and choose to stop trial 0. # We consider trial 1 for early stopping at progression 3, and # choose to stop it. # We consider trial 3 for early stopping at progression 2, and # choose to stop it. early_stopping_strategy = PercentileEarlyStoppingStrategy( percentile_threshold=50, min_curves=3, ) should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices=set(exp.trials.keys()), experiment=exp) self.assertEqual(set(should_stop), {0, 1, 3}) # test case 2, where trial 3 has only 1 data point exp = get_branin_experiment_with_timestamp_map_metric(rate=0.5) for i in range(5): trial = exp.new_trial().add_arm( arm=get_branin_arms(n=1, seed=i)[0]) trial.run() trial.mark_as(status=TrialStatus.COMPLETED) # manually "unalign" timestamps to simulate real-world scenario # where each curve reports results at different steps data = exp.fetch_data() unaligned_timestamps = [0, 1, 4, 1, 2, 3, 1, 3, 4, 0, 1, 2, 0, 2, 4] data.df.loc[data.df["metric_name"] == "branin", "timestamp"] = unaligned_timestamps # manually remove timestamps 1 and 2 for arm 3 data.df.drop([22, 23], inplace=True) exp.attach_data(data=data) """ Dataframe after interpolation: 0 1 2 3 4 timestamp 0 146.138620 NaN NaN 143.375669 65.033535 1 117.388086 113.057480 44.627226 NaN 58.636359 2 111.575393 90.815154 40.237365 NaN 52.239184 3 105.762700 77.324501 35.847504 NaN 48.359101 4 99.950007 NaN 30.522333 NaN 44.479018 """ # We consider trials 0, 2, and 4 for early stopping at progression 4, # and choose to stop trial 0. # We consider trial 1 for early stopping at progression 3, and # choose to stop it. # We consider trial 3 for early stopping at progression 0, and # choose not to stop it. early_stopping_strategy = PercentileEarlyStoppingStrategy( percentile_threshold=50, min_curves=3, ) should_stop = early_stopping_strategy.should_stop_trials_early( trial_indices=set(exp.trials.keys()), experiment=exp) self.assertEqual(set(should_stop), {0, 1})
def test_get_standard_plots(self): exp = get_branin_experiment() self.assertEqual( len( get_standard_plots(experiment=exp, model=get_generation_strategy().model)), 0, ) exp = get_branin_experiment(with_batch=True, minimize=True) exp.trials[0].run() plots = get_standard_plots( experiment=exp, model=Models.BOTORCH(experiment=exp, data=exp.fetch_data()), ) self.assertEqual(len(plots), 6) self.assertTrue(all(isinstance(plot, go.Figure) for plot in plots)) exp = get_branin_experiment_with_multi_objective(with_batch=True) exp.optimization_config.objective.objectives[0].minimize = False exp.optimization_config.objective.objectives[1].minimize = True exp.optimization_config._objective_thresholds = [ ObjectiveThreshold(metric=exp.metrics["branin_a"], op=ComparisonOp.GEQ, bound=-100.0), ObjectiveThreshold(metric=exp.metrics["branin_b"], op=ComparisonOp.LEQ, bound=100.0), ] exp.trials[0].run() plots = get_standard_plots(experiment=exp, model=Models.MOO(experiment=exp, data=exp.fetch_data())) self.assertEqual(len(plots), 7) # All plots are successfully created when objective thresholds are absent exp.optimization_config._objective_thresholds = [] plots = get_standard_plots(experiment=exp, model=Models.MOO(experiment=exp, data=exp.fetch_data())) self.assertEqual(len(plots), 7) exp = get_branin_experiment_with_timestamp_map_metric( with_status_quo=True) exp.new_trial().add_arm(exp.status_quo) exp.trials[0].run() exp.new_trial(generator_run=Models.SOBOL( search_space=exp.search_space).gen(n=1)) exp.trials[1].run() plots = get_standard_plots( experiment=exp, model=Models.BOTORCH(experiment=exp, data=exp.fetch_data()), true_objective_metric_name="branin", ) self.assertEqual(len(plots), 9) self.assertTrue(all(isinstance(plot, go.Figure) for plot in plots)) self.assertIn( "Objective branin_map vs. True Objective Metric branin", [p.layout.title.text for p in plots], ) with self.assertRaisesRegex( ValueError, "Please add a valid true_objective_metric_name"): plots = get_standard_plots( experiment=exp, model=Models.BOTORCH(experiment=exp, data=exp.fetch_data()), true_objective_metric_name="not_present", )