def test_run_packed_artificial_function() -> None: func = MultiExperiment( [ArtificialFunction(name="sphere", block_dimension=2) for _ in range(2)], [100, 100] ) xp = xpbase.Experiment(func, optimizer="OnePlusOne", budget=24, num_workers=2, batch_mode=True, seed=14) summary = xp.run() np.testing.assert_almost_equal(summary["loss"], -9784.8, decimal=1) # makes sure seeding works!
def test_noisy_artificial_function_loss() -> None: func = ArtificialFunction(name="sphere", block_dimension=5, noise_level=0.3) seed = np.random.randint(99999) xp = xpbase.Experiment(func, optimizer="OnePlusOne", budget=5, seed=seed) # Because copy() can have different random initialization for the parameters # The function should be copied early. np.random.seed(seed) pfunc = func.copy() xp.run() loss_ref = xp.result["loss"] # now with copy assert xp._optimizer is not None reco = xp._optimizer.provide_recommendation() assert reco is not None np.testing.assert_equal(pfunc.evaluation_function(reco), loss_ref) np.random.seed(None)
def small_discrete(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]: # prepare list of parameters to sweep for independent variables seedg = create_seed_generator(seed) names = ["hardonemax5", "hardjump5", "hardleadingones5"] optims = sorted(x for x, y in optimizers.registry.items() if "iscrete" in x and "epea" not in x and "DE" not in x and "SSNEA" not in x) functions = [ ArtificialFunction(name, block_dimension=bd, num_blocks=n_blocks, useless_variables=bd * uv_factor * n_blocks) for name in names for bd in [30] for uv_factor in [5, 10] for n_blocks in [1] ] for func in functions: for optim in optims: for budget in [ 100, 400, 700, 1000, 1300, 1600, 1900, 2200, 2500, 2800, 3000 ]: # , 10000]: yield Experiment(func, optim, budget=budget, num_workers=1, seed=next(seedg))
def manyobjective_example( seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]: # prepare list of parameters to sweep for independent variables seedg = create_seed_generator(seed) optims = [ "NaiveTBPSA", "PSO", "DE", "LhsDE", "RandomSearch", "NGO", "Shiva", "DiagonalCMA", "CMA", "OnePlusOne", "TwoPointsDE" ] mofuncs: tp.List[PackedFunctions] = [] name_combinations = itertools.product(["sphere", "cigar"], ["sphere", "hm"], ["sphere", "ellipsoid"], ["rastrigin", "rosenbrock"], ["hm", "rosenbrock"], ["rastrigin", "cigar"]) for names in name_combinations: mofuncs += [ PackedFunctions([ ArtificialFunction(name, block_dimension=6) for name in names ], upper_bounds=np.array( (100, 100, 1000., 7., 300., 500.))) ] for mofunc in mofuncs: for optim in optims: for budget in list(range(100, 5901, 400)): for nw in [1, 100]: yield Experiment(mofunc, optim, budget=budget, num_workers=nw, seed=next(seedg))
def test_pareto_experiment() -> None: # Checking MOO in cross-validation. objective_functions: tp.List[tp.Any] = [ ArtificialFunction("sphere", block_dimension=7), ArtificialFunction("sphere", block_dimension=7), ArtificialFunction("cigar", block_dimension=7), ] xps = helpers.SpecialEvaluationExperiment.create_crossvalidation_experiments( objective_functions, pareto_size=16) assert len(xps) == 15 # 3 xps, multiplied by 5 Pareto extractors param = xps[0].parametrization out = xps[0](*param.args, **param.kwargs) assert isinstance(out, np.ndarray) and out.size == 2 param._losses = out # hack for testing evaluation = xps[0].evaluation_function(param, param, param) assert isinstance(evaluation, float)
def constrained_illconditioned_parallel( seed: Optional[int] = None) -> Iterator[Experiment]: """Many optimizers on ill cond problems with constraints. """ seedg = create_seed_generator(seed) optims = [ "NGO", "Shiva", "DiagonalCMA", "CMA", "PSO", "DE", "MiniDE", "QrDE", "MiniQrDE", "LhsDE", "OnePlusOne", "SQP", "Cobyla", "Powell", "TwoPointsDE", "OnePointDE", "AlmostRotationInvariantDE", "RotationInvariantDE", "Portfolio", "ASCMADEthird", "ASCMADEQRthird", "ASCMA2PDEthird", "CMandAS2", "CMandAS", "CM", "MultiCMA", "TripleCMA", "MultiScaleCMA", "RSQP", "RCobyla", "RPowell", "SQPCMA" ] functions = [ ArtificialFunction(name, block_dimension=50, rotation=rotation) for name in ["cigar", "ellipsoid"] for rotation in [True, False] ] for func in functions: func.parametrization.register_cheap_constraint(_positive_sum) for optim in optims: for function in functions: for budget in [400, 4000, 40000]: yield Experiment(function, optim, budget=budget, num_workers=1, seed=next(seedg))
def doe_dim4( seed: tp.Optional[int] = None ) -> tp.Iterator[ Experiment]: # Here, QR performs best, then Random, then LHS, then Cauchy. # prepare list of parameters to sweep for independent variables seedg = create_seed_generator(seed) names = [ "sphere" ] # n for n in ArtificialFunction.list_sorted_function_names() if "sphere" in n] optims = sorted(x for x, y in optimizers.registry.items() if y.one_shot and "arg" not in x and "mal" not in x) functions = [ ArtificialFunction(name, block_dimension=bd, num_blocks=n_blocks, useless_variables=bd * uv_factor * n_blocks) for name in names for bd in [4] for uv_factor in [0] for n_blocks in [1] ] for func in functions: for optim in optims: for budget in [30, 100, 3000, 10000]: yield Experiment(func, optim, budget=budget, num_workers=1, seed=next(seedg))
def noisy(seed: Optional[int] = None) -> Iterator[Experiment]: """Noisy optimization methods on a few noisy problems. """ seedg = create_seed_generator(seed) optims = ["NGO", "Shiva", "DiagonalCMA"] + sorted( x for x, y in ng.optimizers.registry.items() if ("SPSA" in x or "TBPSA" in x or "ois" in x or "epea" in x or "Random" in x)) for budget in [50000]: for optim in optims: for d in [2, 20, 200]: for name in ["sphere", "rosenbrock"]: for noise_dissymmetry in [False, True]: function = ArtificialFunction( name=name, rotation=True, block_dimension=d, noise_level=10, noise_dissymmetry=noise_dissymmetry, translation_factor=1.0, ) yield Experiment(function, optim, budget=budget, seed=next(seedg))
def harderparallel(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]: """Parallel optimization on 3 classical objective functions.""" seedg = create_seed_generator(seed) names = ["sphere", "rastrigin", "cigar", "ellipsoid"] optims = [ "IsoEMNA", "NaiveIsoEMNA", "AnisoEMNA", "NaiveAnisoEMNA", "CMA", "NaiveTBPSA", "NaiveIsoEMNATBPSA", "IsoEMNATBPSA", "NaiveAnisoEMNATBPSA", "AnisoEMNATBPSA" ] functions = [ ArtificialFunction(name, block_dimension=bd, useless_variables=bd * uv_factor) for name in names for bd in [5, 25] for uv_factor in [0, 5] ] for func in functions: for optim in optims: for budget in [30, 100, 3000, 10000]: for num_workers in [ int(budget / 10), int(budget / 5), int(budget / 3) ]: yield Experiment(func, optim, budget=budget, num_workers=num_workers, seed=next(seedg))
def deceptive(seed: Optional[int] = None) -> Iterator[Experiment]: """Very difficult objective functions: one is highly multimodal (infinitely many local optima), one has an infinite condition number, one has an infinitely long path towards the optimum. Looks somehow fractal.""" seedg = create_seed_generator(seed) names = ["deceptivemultimodal", "deceptiveillcond", "deceptivepath"] optims = [ "NGO", "Shiva", "DiagonalCMA", "PSO", "MiniQrDE", "MiniLhsDE", "MiniDE", "CMA", "QrDE", "DE", "LhsDE" ] functions = [ ArtificialFunction(name, block_dimension=2, num_blocks=n_blocks, rotation=rotation, aggregator=aggregator) for name in names for rotation in [False, True] for n_blocks in [1, 2, 8, 16] for aggregator in ["sum", "max"] ] for func in functions: for optim in optims: for budget in [25, 37, 50, 75, 87] + list(range(100, 20001, 500)): yield Experiment(func, optim, budget=budget, num_workers=1, seed=next(seedg))
def multimodal(seed: tp.Optional[int] = None, para: bool = False) -> tp.Iterator[Experiment]: """Experiment on multimodal functions, namely hm, rastrigin, griewank, rosenbrock, ackley, lunacek, deceptivemultimodal.""" seedg = create_seed_generator(seed) names = ["hm", "rastrigin", "griewank", "rosenbrock", "ackley", "lunacek", "deceptivemultimodal"] # Keep in mind that Rosenbrock is multimodal in high dimension http://ieeexplore.ieee.org/document/6792472/. optims = ["NGO", "Shiva", "DiagonalCMA", "NaiveTBPSA", "TBPSA", "CMA", "PSO", "DE", "MiniDE", "QrDE", "MiniQrDE", "LhsDE", "OnePlusOne", "TwoPointsDE", "OnePointDE", "AlmostRotationInvariantDE", "RotationInvariantDE", "Portfolio", "ASCMADEthird", "ASCMADEQRthird", "ASCMA2PDEthird", "CMandAS2", "CMandAS", "CM", "MultiCMA", "TripleCMA", "MultiScaleCMA"] if not para: optims += ["RSQP", "RCobyla", "RPowell", "SQPCMA", "SQP", "Cobyla", "Powell"] # + list(sorted(x for x, y in ng.optimizers.registry.items() if "chain" in x or "BO" in x)) functions = [ ArtificialFunction(name, block_dimension=bd, useless_variables=bd * uv_factor) for name in names for bd in [3, 25] for uv_factor in [0, 5] ] for func in functions: for optim in optims: for budget in [3000, 10000, 30000, 100000]: for nw in [1000] if para else [1]: yield Experiment(func, optim, budget=budget, num_workers=nw, seed=next(seedg))
def yabbob(seed: tp.Optional[int] = None, parallel: bool = False, big: bool = False, small: bool = False, noise: bool = False, hd: bool = False) -> tp.Iterator[Experiment]: """Yet Another Black-Box Optimization Benchmark. Related to, but without special effort for exactly sticking to, the BBOB/COCO dataset. """ seedg = create_seed_generator(seed) optims = [ "NaiveTBPSA", "TBPSA", "DiagonalCMA", "CMA", "PSO", "DE", "MiniDE", "QrDE", "MiniQrDE", "LhsDE", "OnePlusOne", "TwoPointsDE", "OnePointDE", "AlmostRotationInvariantDE", "RotationInvariantDE", "CMandAS2", "CMandAS" ] if not parallel: optims += ["SQP", "Powell", "chainCMASQP", "chainCMAPowell"] if not parallel and not small: optims += ["Cobyla"] if not small: optims += ["NGO", "Shiva"] # optims += [x for x, y in ng.optimizers.registry.items() if "chain" in x] names = [ "hm", "rastrigin", "griewank", "rosenbrock", "ackley", "lunacek", "deceptivemultimodal", "bucherastrigin", "multipeak" ] names += ["sphere", "doublelinearslope", "stepdoublelinearslope"] names += [ "cigar", "altcigar", "ellipsoid", "altellipsoid", "stepellipsoid", "discus", "bentcigar" ] names += ["deceptiveillcond", "deceptivemultimodal", "deceptivepath"] # Deceptive path is related to the sharp ridge function; there is a long path to the optimum. # Deceptive illcond is related to the difference of powers function; the conditioning varies as we get closer to the optimum. # Deceptive multimodal is related to the Weierstrass function and to the Schaffers function. functions = [ ArtificialFunction(name, block_dimension=d, rotation=rotation, noise_level=100 if noise else 0) for name in names for rotation in [True, False] for num_blocks in [1] for d in ([100, 1000, 3000] if hd else [2, 10, 50]) ] budgets = [50, 200, 800, 3200, 12800] if (big and not noise): budgets = [40000, 80000] elif (small and not noise): budgets = [10, 20, 40] for optim in optims: for function in functions: for budget in budgets: xp = Experiment(function, optim, num_workers=100 if parallel else 1, budget=budget, seed=next(seedg)) if not xp.is_incoherent: yield xp
def test_is_incoherent(optimizer: str, num_workers: int, expected: bool) -> None: func = ArtificialFunction(name="sphere", block_dimension=2) xp = xpbase.Experiment(func, optimizer=optimizer, budget=300, num_workers=num_workers) np.testing.assert_equal(xp.is_incoherent, expected)
def repeated_basic(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]: """Test settings """ seedg = create_seed_generator(seed) function = ArtificialFunction(name="sphere", block_dimension=2, noise_level=1) optims: tp.List[tp.Union[str, ConfiguredOptimizer]] = ["OnePlusOne", optimizers.DifferentialEvolution()] for _ in range(5): for optim in optims: yield Experiment(function, optimizer=optim, num_workers=2, budget=4, seed=next(seedg))
def metanoise(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]: seedg = create_seed_generator(seed) optims = ["NoisyBandit", "TBPSA", "NaiveTBPSA"] for budget in [15, 31, 62, 125, 250, 500, 1000, 2000, 4000, 8000]: for optim in optims: for noise_dissymmetry in [False, True]: function = ArtificialFunction(name="sphere", rotation=True, block_dimension=1, noise_level=10, noise_dissymmetry=noise_dissymmetry, translation_factor=10.) yield Experiment(function, optim, budget=budget, seed=next(seedg))
def basic(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]: """Test settings """ seedg = create_seed_generator(seed) function = ArtificialFunction(name="sphere", block_dimension=2, noise_level=1) np.random.seed(seed) # seed before initializing the function! # initialization uses randomness function.transform_var._initialize() return iter([Experiment(function, optimizer="OnePlusOne", num_workers=2, budget=4, seed=next(seedg))])
def test_run_artificial_function() -> None: func = ArtificialFunction(name="sphere", block_dimension=2) xp = xpbase.Experiment(func, optimizer="OnePlusOne", budget=24, num_workers=2, batch_mode=True, seed=12) summary = xp.run() assert summary["elapsed_time"] < 0.5 # should be much faster np.testing.assert_almost_equal(summary["loss"], 0.08444784112287358) # makes sure seeding works! testing.assert_set_equal(summary.keys(), DESCRIPTION_KEYS) np.testing.assert_equal(summary["elapsed_budget"], 24) np.testing.assert_equal(summary["pseudotime"], 12) # defaults to 1 unit per eval ( /2 because 2 workers)
def illcond(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]: """All optimizers on ill cond problems """ seedg = create_seed_generator(seed) for budget in [500, 1000, 2000, 4000]: for optim in ["SQP", "DE", "CMA", "PSO", "RotationInvariantDE", "NelderMead"]: for rotation in [True, False]: for name in ["ellipsoid", "cigar"]: function = ArtificialFunction(name=name, rotation=rotation, block_dimension=100) yield Experiment(function, optim, budget=budget, seed=next(seedg))
def multiobjective_example(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]: seedg = create_seed_generator(seed) optims = ["NaiveTBPSA", "PSO", "DE", "LhsDE", "RandomSearch", "NGO", "Shiva", "DiagonalCMA", "CMA", "OnePlusOne", "TwoPointsDE"] mofuncs: tp.List[PackedFunctions] = [] for name1 in ["sphere", "cigar"]: for name2 in ["sphere", "cigar", "hm"]: mofuncs += [PackedFunctions([ArtificialFunction(name1, block_dimension=7), ArtificialFunction(name2, block_dimension=7)], upper_bounds=np.array((50., 50.)))] for name3 in ["sphere", "ellipsoid"]: mofuncs += [PackedFunctions([ArtificialFunction(name1, block_dimension=6), ArtificialFunction(name3, block_dimension=6), ArtificialFunction(name2, block_dimension=6)], upper_bounds=np.array((100, 100, 1000.)))] for mofunc in mofuncs: for optim in optims: for budget in list(range(100, 2901, 400)): for nw in [1, 100]: yield Experiment(mofunc, optim, budget=budget, num_workers=nw, seed=next(seedg))
def oneshot2(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]: # Experiment comparing one-shot optimizers in the context of useless vars vs critical vars. seedg = create_seed_generator(seed) names = ["sphere", "altcigar", "cigar", "ellipsoid", "rosenbrock", "rastrigin", "altellipsoid"] optims = sorted(x for x, y in optimizers.registry.items() if y.one_shot and "arg" not in x and "mal" not in x) functions = [ArtificialFunction(name, block_dimension=2, num_blocks=1, useless_variables=20) for name in names] for func in functions: for optim in optims: for budget in [30, 60, 100]: yield Experiment(func, optim, budget=budget, num_workers=1, seed=next(seedg))
def test_functionlib_delayed_job() -> None: np.random.seed(None) func = ArtificialFunction("DelayedSphere", 2) func([0, 0]) # trigger init executor = execution.MockedTimedExecutor(batch_mode=False) x0 = func.transform_var._transforms[0].translation # optimal value job0 = executor.submit(func, x0) job1 = executor.submit(func, x0 + 1.) assert job0.release_time == 0 assert job1.release_time > 0
def spsa_benchmark(seed: Optional[int] = None) -> Iterator[Experiment]: """Some optimizers on a noisy optimization problem. This benchmark is based on the noise benchmark. """ seedg = create_seed_generator(seed) optims = sorted(x for x, y in ng.optimizers.registry.items() if (any(e in x for e in "TBPSA SPSA".split()) and "iscr" not in x)) for budget in [500, 1000, 2000, 4000, 8000, 16000, 32000, 64000, 128000]: for optim in optims: for rotation in [True, False]: for name in ["sphere", "sphere4", "cigar"]: function = ArtificialFunction(name=name, rotation=rotation, block_dimension=20, noise_level=10) yield Experiment(function, optim, budget=budget, seed=next(seedg))
def test_equality() -> None: func = ArtificialFunction(name="sphere", block_dimension=2) xp1 = xpbase.Experiment(func, optimizer="OnePlusOne", budget=300, num_workers=2) xp2 = xpbase.Experiment(func, optimizer="RandomSearch", budget=300, num_workers=2) assert xp1 != xp2
def oneshot3(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]: # General experiment comparing one-shot optimizers, excluding those with "large" or "small" # in the name. seedg = create_seed_generator(seed) names = ["sphere", "altcigar", "cigar", "ellipsoid", "rosenbrock", "rastrigin", "altellipsoid"] optims = sorted(x for x, y in optimizers.registry.items() if y.one_shot and "arg" not in x and "mal" not in x) functions = [ArtificialFunction(name, block_dimension=bd) for name in names for bd in [4, 20]] for func in functions: for optim in optims: for budget in [30, 60, 100]: yield Experiment(func, optim, budget=budget, num_workers=1, seed=next(seedg))
def dim10_select_two_features(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]: # prepare list of parameters to sweep for independent variables seedg = create_seed_generator(seed) names = ["sphere"] optims = sorted(x for x, y in optimizers.registry.items() if y.one_shot and "arg" not in x and "mal" not in x) functions = [ArtificialFunction(name, block_dimension=bd, num_blocks=n_blocks, useless_variables=bd * uv_factor * n_blocks) for name in names for bd in [2] for uv_factor in [5] for n_blocks in [1]] for func in functions: for optim in optims: for budget in [4, 8, 16, 32]: yield Experiment(func, optim, budget=budget, num_workers=1, seed=next(seedg))
def test_run_with_error() -> None: func = ArtificialFunction(name="sphere", block_dimension=2) xp = xpbase.Experiment(func, optimizer="OnePlusOne", budget=300, num_workers=1) with patch("nevergrad.optimization.base.Optimizer.minimize") as run: run.side_effect = ValueError("test error string") with contextlib.redirect_stderr(sys.stdout): summary = xp.run() testing.assert_set_equal(summary.keys(), DESCRIPTION_KEYS) np.testing.assert_equal(summary["error"], "ValueError") assert xp._optimizer is not None np.testing.assert_equal(xp._optimizer.num_tell, 0) # make sure optimizer is kept in case we need to restart (eg.: KeyboardInterrupt) assert not np.isnan(summary["loss"]), "Loss should be recorded with the current recommendation"
def test_sqp_with_constraint() -> None: func = ArtificialFunction("ellipsoid", block_dimension=10, rotation=True, translation_factor=0.1) func.parametrization.register_cheap_constraint( experiments._Constraint("sum", as_bool=True)) xp = Experiment(func, optimizer="ChainMetaModelSQP", budget=150, seed=4290846341) xp._run_with_error()
def noise(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]: """All optimizers on ill cond problems """ seedg = create_seed_generator(seed) optims = sorted(x for x, y in optimizers.registry.items() if ("TBPSA" in x or "ois" in x or "CMA" in x or "epea" in x) and "iscr" not in x) for budget in [500, 1000, 2000, 4000, 8000, 16000, 32000, 64000, 128000]: for optim in optims: for rotation in [True, False]: for name in ["sphere", "cigar", "sphere4"]: function = ArtificialFunction(name=name, rotation=rotation, block_dimension=20, noise_level=10) yield Experiment(function, optim, budget=budget, seed=next(seedg))
def additional_experiment( ): # The signature can also include a seed argument if need be (see experiments.py) funcs = [ ArtificialFunction(name="sphere", block_dimension=10), CustomFunction(2) ] for budget in [10, 100]: for optimizer in ["NewOptimizer", "RandomSearch"]: for func in funcs: # 2 realizations of the same function yield Experiment(func, optimizer=optimizer, budget=budget, num_workers=1)
def illcondi(seed: Optional[int] = None) -> Iterator[Experiment]: """Testing optimizers on ill cond problems. """ seedg = create_seed_generator(seed) optims = ["NGO", "Shiva", "DiagonalCMA", "CMA", "PSO", "DE", "MiniDE", "QrDE", "MiniQrDE", "LhsDE", "OnePlusOne", "SQP", "Cobyla", "Powell", "TwoPointsDE", "OnePointDE", "AlmostRotationInvariantDE", "RotationInvariantDE"] functions = [ ArtificialFunction(name, block_dimension=50, rotation=rotation) for name in ["cigar", "ellipsoid"] for rotation in [True, False] ] for optim in optims: for function in functions: for budget in [100, 1000, 10000]: yield Experiment(function, optim, budget=budget, num_workers=1, seed=next(seedg))