def test_run_packed_artificial_function() -> None:
func = MultiExperiment(
[ArtificialFunction(name="sphere", block_dimension=2) for _ in range(2)], [100, 100]
)
xp = xpbase.Experiment(func, optimizer="OnePlusOne", budget=24, num_workers=2, batch_mode=True, seed=14)
summary = xp.run()
np.testing.assert_almost_equal(summary["loss"], -9784.8, decimal=1) # makes sure seeding works!
def test_noisy_artificial_function_loss() -> None:
func = ArtificialFunction(name="sphere", block_dimension=5, noise_level=0.3)
seed = np.random.randint(99999)
xp = xpbase.Experiment(func, optimizer="OnePlusOne", budget=5, seed=seed)
# Because copy() can have different random initialization for the parameters
# The function should be copied early.
np.random.seed(seed)
pfunc = func.copy()
xp.run()
loss_ref = xp.result["loss"]
# now with copy
assert xp._optimizer is not None
reco = xp._optimizer.provide_recommendation()
assert reco is not None
np.testing.assert_equal(pfunc.evaluation_function(reco), loss_ref)
np.random.seed(None)
def small_discrete(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
# prepare list of parameters to sweep for independent variables
seedg = create_seed_generator(seed)
names = ["hardonemax5", "hardjump5", "hardleadingones5"]
optims = sorted(x for x, y in optimizers.registry.items() if "iscrete" in x
and "epea" not in x and "DE" not in x and "SSNEA" not in x)
functions = [
ArtificialFunction(name,
block_dimension=bd,
num_blocks=n_blocks,
useless_variables=bd * uv_factor * n_blocks)
for name in names for bd in [30] for uv_factor in [5, 10]
for n_blocks in [1]
]
for func in functions:
for optim in optims:
for budget in [
100, 400, 700, 1000, 1300, 1600, 1900, 2200, 2500, 2800,
3000
]: # , 10000]:
yield Experiment(func,
optim,
budget=budget,
num_workers=1,
seed=next(seedg))
def manyobjective_example(
seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
# prepare list of parameters to sweep for independent variables
seedg = create_seed_generator(seed)
optims = [
"NaiveTBPSA", "PSO", "DE", "LhsDE", "RandomSearch", "NGO", "Shiva",
"DiagonalCMA", "CMA", "OnePlusOne", "TwoPointsDE"
]
mofuncs: tp.List[PackedFunctions] = []
name_combinations = itertools.product(["sphere", "cigar"],
["sphere", "hm"],
["sphere", "ellipsoid"],
["rastrigin", "rosenbrock"],
["hm", "rosenbrock"],
["rastrigin", "cigar"])
for names in name_combinations:
mofuncs += [
PackedFunctions([
ArtificialFunction(name, block_dimension=6) for name in names
],
upper_bounds=np.array(
(100, 100, 1000., 7., 300., 500.)))
]
for mofunc in mofuncs:
for optim in optims:
for budget in list(range(100, 5901, 400)):
for nw in [1, 100]:
yield Experiment(mofunc,
optim,
budget=budget,
num_workers=nw,
seed=next(seedg))
def test_pareto_experiment() -> None:
# Checking MOO in cross-validation.
objective_functions: tp.List[tp.Any] = [
ArtificialFunction("sphere", block_dimension=7),
ArtificialFunction("sphere", block_dimension=7),
ArtificialFunction("cigar", block_dimension=7),
]
xps = helpers.SpecialEvaluationExperiment.create_crossvalidation_experiments(
objective_functions, pareto_size=16)
assert len(xps) == 15 # 3 xps, multiplied by 5 Pareto extractors
param = xps[0].parametrization
out = xps[0](*param.args, **param.kwargs)
assert isinstance(out, np.ndarray) and out.size == 2
param._losses = out # hack for testing
evaluation = xps[0].evaluation_function(param, param, param)
assert isinstance(evaluation, float)
def constrained_illconditioned_parallel(
seed: Optional[int] = None) -> Iterator[Experiment]:
"""Many optimizers on ill cond problems with constraints.
"""
seedg = create_seed_generator(seed)
optims = [
"NGO", "Shiva", "DiagonalCMA", "CMA", "PSO", "DE", "MiniDE", "QrDE",
"MiniQrDE", "LhsDE", "OnePlusOne", "SQP", "Cobyla", "Powell",
"TwoPointsDE", "OnePointDE", "AlmostRotationInvariantDE",
"RotationInvariantDE", "Portfolio", "ASCMADEthird", "ASCMADEQRthird",
"ASCMA2PDEthird", "CMandAS2", "CMandAS", "CM", "MultiCMA", "TripleCMA",
"MultiScaleCMA", "RSQP", "RCobyla", "RPowell", "SQPCMA"
]
functions = [
ArtificialFunction(name, block_dimension=50, rotation=rotation)
for name in ["cigar", "ellipsoid"] for rotation in [True, False]
]
for func in functions:
func.parametrization.register_cheap_constraint(_positive_sum)
for optim in optims:
for function in functions:
for budget in [400, 4000, 40000]:
yield Experiment(function,
optim,
budget=budget,
num_workers=1,
seed=next(seedg))
def doe_dim4(
seed: tp.Optional[int] = None
) -> tp.Iterator[
Experiment]: # Here, QR performs best, then Random, then LHS, then Cauchy.
# prepare list of parameters to sweep for independent variables
seedg = create_seed_generator(seed)
names = [
"sphere"
] # n for n in ArtificialFunction.list_sorted_function_names() if "sphere" in n]
optims = sorted(x for x, y in optimizers.registry.items()
if y.one_shot and "arg" not in x and "mal" not in x)
functions = [
ArtificialFunction(name,
block_dimension=bd,
num_blocks=n_blocks,
useless_variables=bd * uv_factor * n_blocks)
for name in names for bd in [4] for uv_factor in [0]
for n_blocks in [1]
]
for func in functions:
for optim in optims:
for budget in [30, 100, 3000, 10000]:
yield Experiment(func,
optim,
budget=budget,
num_workers=1,
seed=next(seedg))
def noisy(seed: Optional[int] = None) -> Iterator[Experiment]:
"""Noisy optimization methods on a few noisy problems.
"""
seedg = create_seed_generator(seed)
optims = ["NGO", "Shiva", "DiagonalCMA"] + sorted(
x for x, y in ng.optimizers.registry.items()
if ("SPSA" in x or "TBPSA" in x or "ois" in x or "epea" in x
or "Random" in x))
for budget in [50000]:
for optim in optims:
for d in [2, 20, 200]:
for name in ["sphere", "rosenbrock"]:
for noise_dissymmetry in [False, True]:
function = ArtificialFunction(
name=name,
rotation=True,
block_dimension=d,
noise_level=10,
noise_dissymmetry=noise_dissymmetry,
translation_factor=1.0,
)
yield Experiment(function,
optim,
budget=budget,
seed=next(seedg))
def harderparallel(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
"""Parallel optimization on 3 classical objective functions."""
seedg = create_seed_generator(seed)
names = ["sphere", "rastrigin", "cigar", "ellipsoid"]
optims = [
"IsoEMNA", "NaiveIsoEMNA", "AnisoEMNA", "NaiveAnisoEMNA", "CMA",
"NaiveTBPSA", "NaiveIsoEMNATBPSA", "IsoEMNATBPSA",
"NaiveAnisoEMNATBPSA", "AnisoEMNATBPSA"
]
functions = [
ArtificialFunction(name,
block_dimension=bd,
useless_variables=bd * uv_factor) for name in names
for bd in [5, 25] for uv_factor in [0, 5]
]
for func in functions:
for optim in optims:
for budget in [30, 100, 3000, 10000]:
for num_workers in [
int(budget / 10),
int(budget / 5),
int(budget / 3)
]:
yield Experiment(func,
optim,
budget=budget,
num_workers=num_workers,
seed=next(seedg))
def deceptive(seed: Optional[int] = None) -> Iterator[Experiment]:
"""Very difficult objective functions: one is highly multimodal (infinitely many local optima),
one has an infinite condition number, one has an infinitely long path towards the optimum.
Looks somehow fractal."""
seedg = create_seed_generator(seed)
names = ["deceptivemultimodal", "deceptiveillcond", "deceptivepath"]
optims = [
"NGO", "Shiva", "DiagonalCMA", "PSO", "MiniQrDE", "MiniLhsDE",
"MiniDE", "CMA", "QrDE", "DE", "LhsDE"
]
functions = [
ArtificialFunction(name,
block_dimension=2,
num_blocks=n_blocks,
rotation=rotation,
aggregator=aggregator) for name in names
for rotation in [False, True] for n_blocks in [1, 2, 8, 16]
for aggregator in ["sum", "max"]
]
for func in functions:
for optim in optims:
for budget in [25, 37, 50, 75, 87] + list(range(100, 20001, 500)):
yield Experiment(func,
optim,
budget=budget,
num_workers=1,
seed=next(seedg))
def multimodal(seed: tp.Optional[int] = None, para: bool = False) -> tp.Iterator[Experiment]:
"""Experiment on multimodal functions, namely hm, rastrigin, griewank, rosenbrock, ackley, lunacek,
deceptivemultimodal."""
seedg = create_seed_generator(seed)
names = ["hm", "rastrigin", "griewank", "rosenbrock", "ackley", "lunacek", "deceptivemultimodal"]
# Keep in mind that Rosenbrock is multimodal in high dimension http://ieeexplore.ieee.org/document/6792472/.
optims = ["NGO", "Shiva", "DiagonalCMA", "NaiveTBPSA", "TBPSA",
"CMA", "PSO", "DE", "MiniDE", "QrDE", "MiniQrDE", "LhsDE", "OnePlusOne",
"TwoPointsDE", "OnePointDE", "AlmostRotationInvariantDE", "RotationInvariantDE",
"Portfolio", "ASCMADEthird", "ASCMADEQRthird", "ASCMA2PDEthird", "CMandAS2", "CMandAS", "CM",
"MultiCMA", "TripleCMA", "MultiScaleCMA"]
if not para:
optims += ["RSQP", "RCobyla", "RPowell", "SQPCMA", "SQP", "Cobyla", "Powell"]
# + list(sorted(x for x, y in ng.optimizers.registry.items() if "chain" in x or "BO" in x))
functions = [
ArtificialFunction(name, block_dimension=bd, useless_variables=bd * uv_factor)
for name in names
for bd in [3, 25]
for uv_factor in [0, 5]
]
for func in functions:
for optim in optims:
for budget in [3000, 10000, 30000, 100000]:
for nw in [1000] if para else [1]:
yield Experiment(func, optim, budget=budget, num_workers=nw, seed=next(seedg))
def yabbob(seed: tp.Optional[int] = None,
parallel: bool = False,
big: bool = False,
small: bool = False,
noise: bool = False,
hd: bool = False) -> tp.Iterator[Experiment]:
"""Yet Another Black-Box Optimization Benchmark.
Related to, but without special effort for exactly sticking to, the BBOB/COCO dataset.
"""
seedg = create_seed_generator(seed)
optims = [
"NaiveTBPSA", "TBPSA", "DiagonalCMA", "CMA", "PSO", "DE", "MiniDE",
"QrDE", "MiniQrDE", "LhsDE", "OnePlusOne", "TwoPointsDE", "OnePointDE",
"AlmostRotationInvariantDE", "RotationInvariantDE", "CMandAS2",
"CMandAS"
]
if not parallel:
optims += ["SQP", "Powell", "chainCMASQP", "chainCMAPowell"]
if not parallel and not small:
optims += ["Cobyla"]
if not small:
optims += ["NGO", "Shiva"]
# optims += [x for x, y in ng.optimizers.registry.items() if "chain" in x]
names = [
"hm", "rastrigin", "griewank", "rosenbrock", "ackley", "lunacek",
"deceptivemultimodal", "bucherastrigin", "multipeak"
]
names += ["sphere", "doublelinearslope", "stepdoublelinearslope"]
names += [
"cigar", "altcigar", "ellipsoid", "altellipsoid", "stepellipsoid",
"discus", "bentcigar"
]
names += ["deceptiveillcond", "deceptivemultimodal", "deceptivepath"]
# Deceptive path is related to the sharp ridge function; there is a long path to the optimum.
# Deceptive illcond is related to the difference of powers function; the conditioning varies as we get closer to the optimum.
# Deceptive multimodal is related to the Weierstrass function and to the Schaffers function.
functions = [
ArtificialFunction(name,
block_dimension=d,
rotation=rotation,
noise_level=100 if noise else 0) for name in names
for rotation in [True, False] for num_blocks in [1]
for d in ([100, 1000, 3000] if hd else [2, 10, 50])
]
budgets = [50, 200, 800, 3200, 12800]
if (big and not noise):
budgets = [40000, 80000]
elif (small and not noise):
budgets = [10, 20, 40]
for optim in optims:
for function in functions:
for budget in budgets:
xp = Experiment(function,
optim,
num_workers=100 if parallel else 1,
budget=budget,
seed=next(seedg))
if not xp.is_incoherent:
yield xp
def test_is_incoherent(optimizer: str, num_workers: int,
expected: bool) -> None:
func = ArtificialFunction(name="sphere", block_dimension=2)
xp = xpbase.Experiment(func,
optimizer=optimizer,
budget=300,
num_workers=num_workers)
np.testing.assert_equal(xp.is_incoherent, expected)
def repeated_basic(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
"""Test settings
"""
seedg = create_seed_generator(seed)
function = ArtificialFunction(name="sphere", block_dimension=2, noise_level=1)
optims: tp.List[tp.Union[str, ConfiguredOptimizer]] = ["OnePlusOne", optimizers.DifferentialEvolution()]
for _ in range(5):
for optim in optims:
yield Experiment(function, optimizer=optim, num_workers=2, budget=4, seed=next(seedg))
def metanoise(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
seedg = create_seed_generator(seed)
optims = ["NoisyBandit", "TBPSA", "NaiveTBPSA"]
for budget in [15, 31, 62, 125, 250, 500, 1000, 2000, 4000, 8000]:
for optim in optims:
for noise_dissymmetry in [False, True]:
function = ArtificialFunction(name="sphere", rotation=True, block_dimension=1, noise_level=10,
noise_dissymmetry=noise_dissymmetry, translation_factor=10.)
yield Experiment(function, optim, budget=budget, seed=next(seedg))
def basic(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
"""Test settings
"""
seedg = create_seed_generator(seed)
function = ArtificialFunction(name="sphere", block_dimension=2, noise_level=1)
np.random.seed(seed) # seed before initializing the function!
# initialization uses randomness
function.transform_var._initialize()
return iter([Experiment(function, optimizer="OnePlusOne", num_workers=2, budget=4, seed=next(seedg))])
def test_run_artificial_function() -> None:
func = ArtificialFunction(name="sphere", block_dimension=2)
xp = xpbase.Experiment(func, optimizer="OnePlusOne", budget=24, num_workers=2, batch_mode=True, seed=12)
summary = xp.run()
assert summary["elapsed_time"] < 0.5 # should be much faster
np.testing.assert_almost_equal(summary["loss"], 0.08444784112287358) # makes sure seeding works!
testing.assert_set_equal(summary.keys(), DESCRIPTION_KEYS)
np.testing.assert_equal(summary["elapsed_budget"], 24)
np.testing.assert_equal(summary["pseudotime"], 12) # defaults to 1 unit per eval ( /2 because 2 workers)
def illcond(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
"""All optimizers on ill cond problems
"""
seedg = create_seed_generator(seed)
for budget in [500, 1000, 2000, 4000]:
for optim in ["SQP", "DE", "CMA", "PSO", "RotationInvariantDE", "NelderMead"]:
for rotation in [True, False]:
for name in ["ellipsoid", "cigar"]:
function = ArtificialFunction(name=name, rotation=rotation, block_dimension=100)
yield Experiment(function, optim, budget=budget, seed=next(seedg))
def multiobjective_example(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
seedg = create_seed_generator(seed)
optims = ["NaiveTBPSA", "PSO", "DE", "LhsDE", "RandomSearch", "NGO", "Shiva", "DiagonalCMA", "CMA", "OnePlusOne", "TwoPointsDE"]
mofuncs: tp.List[PackedFunctions] = []
for name1 in ["sphere", "cigar"]:
for name2 in ["sphere", "cigar", "hm"]:
mofuncs += [PackedFunctions([ArtificialFunction(name1, block_dimension=7),
ArtificialFunction(name2, block_dimension=7)],
upper_bounds=np.array((50., 50.)))]
for name3 in ["sphere", "ellipsoid"]:
mofuncs += [PackedFunctions([ArtificialFunction(name1, block_dimension=6),
ArtificialFunction(name3, block_dimension=6),
ArtificialFunction(name2, block_dimension=6)],
upper_bounds=np.array((100, 100, 1000.)))]
for mofunc in mofuncs:
for optim in optims:
for budget in list(range(100, 2901, 400)):
for nw in [1, 100]:
yield Experiment(mofunc, optim, budget=budget, num_workers=nw, seed=next(seedg))
def oneshot2(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
# Experiment comparing one-shot optimizers in the context of useless vars vs critical vars.
seedg = create_seed_generator(seed)
names = ["sphere", "altcigar", "cigar", "ellipsoid", "rosenbrock", "rastrigin", "altellipsoid"]
optims = sorted(x for x, y in optimizers.registry.items() if y.one_shot and "arg" not in x and "mal" not in x)
functions = [ArtificialFunction(name, block_dimension=2, num_blocks=1, useless_variables=20) for name in names]
for func in functions:
for optim in optims:
for budget in [30, 60, 100]:
yield Experiment(func, optim, budget=budget, num_workers=1, seed=next(seedg))
def test_functionlib_delayed_job() -> None:
np.random.seed(None)
func = ArtificialFunction("DelayedSphere", 2)
func([0, 0]) # trigger init
executor = execution.MockedTimedExecutor(batch_mode=False)
x0 = func.transform_var._transforms[0].translation # optimal value
job0 = executor.submit(func, x0)
job1 = executor.submit(func, x0 + 1.)
assert job0.release_time == 0
assert job1.release_time > 0
def spsa_benchmark(seed: Optional[int] = None) -> Iterator[Experiment]:
"""Some optimizers on a noisy optimization problem. This benchmark is based on the noise benchmark.
"""
seedg = create_seed_generator(seed)
optims = sorted(x for x, y in ng.optimizers.registry.items() if (any(e in x for e in "TBPSA SPSA".split()) and "iscr" not in x))
for budget in [500, 1000, 2000, 4000, 8000, 16000, 32000, 64000, 128000]:
for optim in optims:
for rotation in [True, False]:
for name in ["sphere", "sphere4", "cigar"]:
function = ArtificialFunction(name=name, rotation=rotation, block_dimension=20, noise_level=10)
yield Experiment(function, optim, budget=budget, seed=next(seedg))
def test_equality() -> None:
func = ArtificialFunction(name="sphere", block_dimension=2)
xp1 = xpbase.Experiment(func,
optimizer="OnePlusOne",
budget=300,
num_workers=2)
xp2 = xpbase.Experiment(func,
optimizer="RandomSearch",
budget=300,
num_workers=2)
assert xp1 != xp2
def oneshot3(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
# General experiment comparing one-shot optimizers, excluding those with "large" or "small"
# in the name.
seedg = create_seed_generator(seed)
names = ["sphere", "altcigar", "cigar", "ellipsoid", "rosenbrock", "rastrigin", "altellipsoid"]
optims = sorted(x for x, y in optimizers.registry.items() if y.one_shot and "arg" not in x and "mal" not in x)
functions = [ArtificialFunction(name, block_dimension=bd) for name in names for bd in [4, 20]]
for func in functions:
for optim in optims:
for budget in [30, 60, 100]:
yield Experiment(func, optim, budget=budget, num_workers=1, seed=next(seedg))
def dim10_select_two_features(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
# prepare list of parameters to sweep for independent variables
seedg = create_seed_generator(seed)
names = ["sphere"]
optims = sorted(x for x, y in optimizers.registry.items() if y.one_shot and "arg" not in x and "mal" not in x)
functions = [ArtificialFunction(name, block_dimension=bd, num_blocks=n_blocks, useless_variables=bd * uv_factor * n_blocks)
for name in names for bd in [2] for uv_factor in [5] for n_blocks in [1]]
for func in functions:
for optim in optims:
for budget in [4, 8, 16, 32]:
yield Experiment(func, optim, budget=budget, num_workers=1, seed=next(seedg))
def test_run_with_error() -> None:
func = ArtificialFunction(name="sphere", block_dimension=2)
xp = xpbase.Experiment(func, optimizer="OnePlusOne", budget=300, num_workers=1)
with patch("nevergrad.optimization.base.Optimizer.minimize") as run:
run.side_effect = ValueError("test error string")
with contextlib.redirect_stderr(sys.stdout):
summary = xp.run()
testing.assert_set_equal(summary.keys(), DESCRIPTION_KEYS)
np.testing.assert_equal(summary["error"], "ValueError")
assert xp._optimizer is not None
np.testing.assert_equal(xp._optimizer.num_tell, 0) # make sure optimizer is kept in case we need to restart (eg.: KeyboardInterrupt)
assert not np.isnan(summary["loss"]), "Loss should be recorded with the current recommendation"
def test_sqp_with_constraint() -> None:
func = ArtificialFunction("ellipsoid",
block_dimension=10,
rotation=True,
translation_factor=0.1)
func.parametrization.register_cheap_constraint(
experiments._Constraint("sum", as_bool=True))
xp = Experiment(func,
optimizer="ChainMetaModelSQP",
budget=150,
seed=4290846341)
xp._run_with_error()
def noise(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
"""All optimizers on ill cond problems
"""
seedg = create_seed_generator(seed)
optims = sorted(x for x, y in optimizers.registry.items()
if ("TBPSA" in x or "ois" in x or "CMA" in x or "epea" in x) and "iscr" not in x)
for budget in [500, 1000, 2000, 4000, 8000, 16000, 32000, 64000, 128000]:
for optim in optims:
for rotation in [True, False]:
for name in ["sphere", "cigar", "sphere4"]:
function = ArtificialFunction(name=name, rotation=rotation, block_dimension=20, noise_level=10)
yield Experiment(function, optim, budget=budget, seed=next(seedg))
def additional_experiment(
): # The signature can also include a seed argument if need be (see experiments.py)
funcs = [
ArtificialFunction(name="sphere", block_dimension=10),
CustomFunction(2)
]
for budget in [10, 100]:
for optimizer in ["NewOptimizer", "RandomSearch"]:
for func in funcs: # 2 realizations of the same function
yield Experiment(func,
optimizer=optimizer,
budget=budget,
num_workers=1)
def illcondi(seed: Optional[int] = None) -> Iterator[Experiment]:
"""Testing optimizers on ill cond problems.
"""
seedg = create_seed_generator(seed)
optims = ["NGO", "Shiva", "DiagonalCMA", "CMA", "PSO", "DE", "MiniDE", "QrDE", "MiniQrDE", "LhsDE", "OnePlusOne", "SQP", "Cobyla",
"Powell", "TwoPointsDE", "OnePointDE", "AlmostRotationInvariantDE", "RotationInvariantDE"]
functions = [
ArtificialFunction(name, block_dimension=50, rotation=rotation) for name in ["cigar", "ellipsoid"] for rotation in [True, False]
]
for optim in optims:
for function in functions:
for budget in [100, 1000, 10000]:
yield Experiment(function, optim, budget=budget, num_workers=1, seed=next(seedg))
