def test_tell_not_asked(name: str) -> None: param = ng.p.Scalar() with testing.suppress_nevergrad_warnings(): opt = optlib.registry[name](parametrization=param, budget=2, num_workers=2) opt.llambda = 2 # type: ignore t_10 = opt.parametrization.spawn_child(new_value=10) t_100 = opt.parametrization.spawn_child(new_value=100) assert not opt.population # type: ignore opt.tell(t_10, 90) # not asked assert len(opt.population) == 1 # type: ignore asked = opt.ask() opt.tell(asked, 88) assert len(opt.population) == 2 # type: ignore opt.tell(t_100, 0) # not asked asked = opt.ask() opt.tell(asked, 89) assert len(opt.population) == 2 # type: ignore assert opt.num_tell == 4, opt.num_tell assert opt.num_ask == 2 assert len(opt.population) == 2 # type: ignore assert int(opt.recommend().value) == 100 if isinstance(opt.population, dict): # type: ignore assert t_100.uid in opt.population # type: ignore for point, value in opt.archive.items_as_arrays(): assert value.count == 1, f"Error for point {point}"
def check_optimizer( optimizer_cls: tp.Union[base.ConfiguredOptimizer, tp.Type[base.Optimizer]], budget: int = 300, verify_value: bool = True, ) -> None: # recast optimizer do not support num_workers > 1, and respect no_parallelization. num_workers = 1 if optimizer_cls.recast or optimizer_cls.no_parallelization else 2 num_attempts = 1 if not verify_value else 3 # allow 3 attemps to get to the optimum (shit happens...) optimum = [0.5, -0.8] fitness = Fitness(optimum) for k in range(1, num_attempts + 1): fitness = Fitness(optimum) optimizer = optimizer_cls(parametrization=len(optimum), budget=budget, num_workers=num_workers) assert isinstance( optimizer.provide_recommendation(), ng.p.Parameter), "Recommendation should be available from start" with testing.suppress_nevergrad_warnings(): candidate = optimizer.minimize(fitness) raised = False if verify_value: try: np.testing.assert_array_almost_equal(candidate.args[0], optimum, decimal=1) except AssertionError as e: raised = True print( f"Attemp #{k}: failed with best point {tuple(candidate.args[0])}" ) if k == num_attempts: raise e if not raised: break if budget > 100: slope, intercept = fitness.get_factors() print(f"For your information: slope={slope} and intercept={intercept}") # make sure we are correctly tracking the best values archive = optimizer.archive assert optimizer.current_bests[ "pessimistic"].pessimistic_confidence_bound == min( v.pessimistic_confidence_bound for v in archive.values()) # add a random point to test tell_not_asked assert not optimizer._asked, "All `ask`s should have been followed by a `tell`" try: data = np.random.normal(0, 1, size=optimizer.dimension) candidate = optimizer.parametrization.spawn_child( ).set_standardized_data(data) optimizer.tell(candidate, 12.0) except Exception as e: # pylint: disable=broad-except if not isinstance(e, base.errors.TellNotAskedNotSupportedError): raise AssertionError( "Optimizers should raise base.TellNotAskedNotSupportedError " "at when telling unasked points if they do not support it" ) from e else: assert optimizer.num_tell == budget + 1 assert optimizer.num_tell_not_asked == 1
def test_optimizers_recommendation(name: str, recomkeeper: RecommendationKeeper) -> None: if name in UNSEEDABLE: raise SkipTest("Not playing nicely with the tests (unseedable)") if "BO" in name: raise SkipTest("BO differs from one computer to another") # set up environment optimizer_cls = registry[name] np.random.seed(None) if optimizer_cls.recast: np.random.seed(12) random.seed(12) # may depend on non numpy generator # budget=6 by default, larger for special cases needing more budget = { "WidePSO": 100, "PSO": 200, "MEDA": 100, "EDA": 100, "MPCEDA": 100, "TBPSA": 100 }.get(name, 6) if isinstance(optimizer_cls, (optlib.DifferentialEvolution, optlib.EvolutionStrategy)): budget = 80 dimension = min(16, max(4, int(np.sqrt(budget)))) # set up problem fitness = Fitness([0.5, -0.8, 0, 4] + (5 * np.cos(np.arange(dimension - 4))).tolist()) with testing.suppress_nevergrad_warnings(): optim = optimizer_cls(parametrization=dimension, budget=budget, num_workers=1) optim.parametrization.random_state.seed(12) np.testing.assert_equal(optim.name, name) # the following context manager speeds up BO tests # BEWARE: BO tests are deterministic but can get different results from a computer to another. # Reducing the precision could help in this regard. # patched = partial(acq_max, n_warmup=10000, n_iter=2) # with patch("bayes_opt.bayesian_optimization.acq_max", patched): recom = optim.minimize(fitness) if name not in recomkeeper.recommendations.index: recomkeeper.recommendations.loc[name, :dimension] = tuple(recom.value) raise ValueError( f'Recorded the value {tuple(recom.value)} for optimizer "{name}", please rerun this test locally.' ) # BO slightly differs from a computer to another decimal = 2 if isinstance(optimizer_cls, optlib.ParametrizedBO) or "BO" in name else 5 np.testing.assert_array_almost_equal( recom.value, recomkeeper.recommendations.loc[name, :][:dimension], decimal=decimal, err_msg="Something has changed, if this is normal, delete the following " f"file and rerun to update the values:\n{recomkeeper.filepath}", ) # check that by default the recommendation has been evaluated if isinstance(optimizer_cls, optlib.EvolutionStrategy): # no noisy variants assert recom.loss is not None
def test_bo_ordering() -> None: with testing.suppress_nevergrad_warnings(): # tests do not need to be efficient optim = ng.optimizers.ParametrizedBO(initialization="Hammersley")( parametrization=ng.p.Choice(range(12)), budget=10 ) cand = optim.ask() optim.tell(cand, 12) optim.provide_recommendation()
def test_parametrization_offset(name: str) -> None: if "PSO" in name or "BO" in name: raise SkipTest("PSO and BO have large initial variance") if "Cobyla" in name and platform.system() == "Windows": raise SkipTest("Cobyla is flaky on Windows for unknown reasons") parametrization = ng.p.Instrumentation(ng.p.Array(init=[1e12, 1e12])) with testing.suppress_nevergrad_warnings(): optimizer = registry[name](parametrization, budget=100, num_workers=1) for k in range(10 if "BO" not in name else 2): candidate = optimizer.ask() assert ( candidate.args[0][0] > 100 ), f"Candidate value[0] at iteration #{k} is below 100: {candidate.value}" optimizer.tell(candidate, 0)