def test_commandline_launch() -> None:
with tempfile.TemporaryDirectory() as folder:
output = Path(folder) / "benchmark_launch_test.csv"
# commandline test
# TODO make it work on Windows!
# TODO make it work again on the CI (Linux), this started failing with #630 for no reason
with testing.skip_error_on_systems(FailedJobError, systems=("Windows", "Linux")):
CommandFunction(
command=[
sys.executable,
"-m",
"nevergrad.benchmark",
"additional_experiment",
"--cap_index",
"2",
"--num_workers",
"2",
"--output",
str(output),
"--imports",
str(Path(__file__).parent / "additional" / "example.py"),
]
)()
assert output.exists()
df = utils.Selector.read_csv(str(output))
testing.assert_set_equal(
df.columns, DESCRIPTION_KEYS | {"offset"}
) # "offset" comes from the custom function
np.testing.assert_equal(len(df), 2)
def unique(
self, column_s: tp.Union[str, tp.Sequence[str]]
) -> tp.Union[tp.Tuple[tp.Any, ...], tp.Set[tp.Tuple[tp.Any, ...]]]:
"""Returns the set of unique values or set of values for a column or columns
Parameter
---------
column_s: str or tp.Sequence[str]
a column name, or list of column names
Returns
-------
set
a set of values if the input was a column name, or a set of tuple of values
if the name was a list of columns
"""
if isinstance(column_s, str):
return set(self.loc[:,
column_s]) # equivalent to df.<name>.unique()
elif isinstance(column_s, (list, tuple)):
testing.assert_set_equal(set(column_s) - set(self.columns), {},
err_msg="Unknown column(s)")
df = self.loc[:, column_s]
assert not df.isnull().values.any(), "Cannot work with NaN values"
return set(tuple(row) for row in df.itertuples(index=False))
else:
raise NotImplementedError(
"Only strings, lists and tuples are allowed")
def __call__(self, **kwargs: tp.Any) -> str:
testing.assert_set_equal(kwargs,
self.parameters,
err_msg="Wrong input parameters.")
return Placeholder.sub(self._text,
self.filepath.suffix,
replacers=kwargs)
def test_optimization_doc_parametrization_example() -> None:
instrum = ng.p.Instrumentation(ng.p.Array(shape=(2, )), y=ng.p.Scalar())
optimizer = optlib.OnePlusOne(parametrization=instrum, budget=100)
recom = optimizer.minimize(_square)
assert len(recom.args) == 1
testing.assert_set_equal(recom.kwargs, ["y"])
value = _square(*recom.args, **recom.kwargs)
assert value < 0.2 # should be large enough by an order of magnitude
def test_run_artificial_function() -> None:
func = ArtificialFunction(name="sphere", block_dimension=2)
xp = xpbase.Experiment(func, optimizer="OnePlusOne", budget=24, num_workers=2, batch_mode=True, seed=12)
summary = xp.run()
assert summary["elapsed_time"] < 0.5 # should be much faster
np.testing.assert_almost_equal(summary["loss"], 0.08444784112287358) # makes sure seeding works!
testing.assert_set_equal(summary.keys(), DESCRIPTION_KEYS)
np.testing.assert_equal(summary["elapsed_budget"], 24)
np.testing.assert_equal(summary["pseudotime"], 12) # defaults to 1 unit per eval ( /2 because 2 workers)
def test_launch() -> None:
with tempfile.TemporaryDirectory() as folder:
with patch("nevergrad.benchmark.plotting.create_plots"): # dont plot
output = Path(folder) / "benchmark_launch_test.csv"
# commandline test
repeated_launch("repeated_basic", cap_index=4, num_workers=2, output=output, plot=True)
assert output.exists()
df = utils.Selector.read_csv(str(output))
testing.assert_set_equal(df.unique("optimizer_name"), {"DifferentialEvolution()", "OnePlusOne"})
assert isinstance(df, utils.Selector)
np.testing.assert_equal(len(df), 4)
def test_run_with_error() -> None:
func = ArtificialFunction(name="sphere", block_dimension=2)
xp = xpbase.Experiment(func, optimizer="OnePlusOne", budget=300, num_workers=1)
with patch("nevergrad.optimization.base.Optimizer.minimize") as run:
run.side_effect = ValueError("test error string")
with contextlib.redirect_stderr(sys.stdout):
summary = xp.run()
testing.assert_set_equal(summary.keys(), DESCRIPTION_KEYS)
np.testing.assert_equal(summary["error"], "ValueError")
assert xp._optimizer is not None
np.testing.assert_equal(xp._optimizer.num_tell, 0) # make sure optimizer is kept in case we need to restart (eg.: KeyboardInterrupt)
assert not np.isnan(summary["loss"]), "Loss should be recorded with the current recommendation"
def test_experiment_chunk_split() -> None:
chunk = core.BenchmarkChunk(name="repeated_basic", seed=12, repetitions=2)
chunks = chunk.split(2)
chunks = [chunks[0]] + chunks[1].split(3)
np.testing.assert_array_equal([len(c) for c in chunks], [10, 4, 3, 3])
chained = [x[0] for x in itertools.chain.from_iterable(chunks)]
# check full order (everythink only once)
np.testing.assert_array_equal(
chained,
[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 1, 7, 13, 19, 3, 9, 15, 5, 11, 17])
testing.assert_set_equal(chained, range(20))
assert chunks[0].id.endswith("_i0m2"), f"Wrong id {chunks[0].id}"
assert chunks[0].id[:-4] == chunk.id[:-4], "Id prefix should be inherited"
def test_xp_plotter() -> None:
opt = "OnePlusOneOptimizer"
df = utils.Selector.read_csv(Path(__file__).parent / "sphere_perf_example.csv").select(optimizer_name=[opt])
data = plotting.XpPlotter.make_data(df)
# check data
testing.assert_set_equal(data.keys(), {opt})
testing.assert_set_equal(data[opt].keys(), {"budget", "loss", "loss_std", "num_eval"})
np.testing.assert_almost_equal(data[opt]["budget"], [200, 400, 800])
np.testing.assert_almost_equal(data[opt]["loss"], [0.4811605, 0.3920045, 0.14778369])
np.testing.assert_almost_equal(data[opt]["loss_std"], [0.83034832, 0.73255529, 0.18551625])
# plot
with patch('matplotlib.pyplot.Figure.tight_layout'): # avoid warning message
plotter = plotting.XpPlotter(data, title="Title")
with patch('matplotlib.pyplot.Figure.savefig'):
plotter.save("should_not_exist.png")
def assert_equivalent(self, other: pd.DataFrame, err_msg: str = "") -> None:
"""Asserts that two selectors are equal, up to row and column permutations
Note
----
Use sparsely, since it is quite slow to test
"""
testing.assert_set_equal(other.columns, self.columns, f"Different columns\n{err_msg}")
np.testing.assert_equal(len(other), len(self), "Different number of rows\n{err_msg}")
other_df = other.loc[:, self.columns]
df_rows: tp.List[tp.List[tp.Tuple[tp.Any, ...]]] = [[], []]
for k, df in enumerate([self, other_df]):
for row in df.itertuples(index=False):
df_rows[k].append(tuple(row))
df_rows[k].sort()
for row1, row2 in zip(*df_rows):
np.testing.assert_array_equal(row1, row2, err_msg=err_msg)
def test_pruning() -> None:
param = ng.p.Scalar(init=12.0)
archive: utils.Archive[utils.MultiValue] = utils.Archive()
for k in range(3):
value = utils.MultiValue(param, float(k), reference=param)
archive[(float(k),)] = value
value = utils.MultiValue(param, 1.0, reference=param)
value.add_evaluation(1.0)
archive[(3.0,)] = value
# pruning
pruning = utils.Pruning(min_len=1, max_len=3)
# 0 is best optimistic and average, and 3 is best pessimistic (variance=0)
archive = pruning(archive)
testing.assert_set_equal([x[0] for x in archive.keys_as_arrays()], [0, 3], err_msg=f"Repetition #{k+1}")
pickle.dumps(archive) # should be picklable
# should not change anything this time
archive2 = pruning(archive)
testing.assert_set_equal([x[0] for x in archive2.keys_as_arrays()], [0, 3], err_msg=f"Repetition #{k+1}")
def instantiate_to_folder(self, outfolder: tp.Union[Path, str],
kwargs: tp.Dict[str, tp.Any]) -> None:
testing.assert_set_equal(kwargs, {x.name
for x in self.placeholders},
err_msg="Wrong input parameters.")
outfolder = Path(outfolder).expanduser().absolute()
assert outfolder != self.folder, "Do not instantiate on same folder!"
symlink_folder_tree(self.folder, outfolder)
for file_func in self.file_functions:
inst_fp = outfolder / file_func.filepath.relative_to(self.folder)
os.remove(str(
inst_fp)) # remove symlink to avoid writing in original dir
with inst_fp.open("w") as f:
f.write(
file_func(
**{
x: y
for x, y in kwargs.items()
if x in file_func.parameters
}))
def test_selector(criteria: tp.Any, expected: tp.List[str]) -> None:
df = utils.Selector(index=["i1", "i2", "i3"], columns=["c1", "c2"])
for i, c in itertools.product(df.index, df.columns):
df.loc[i, c] = f"{i}-{c}"
df_select = df.select(**criteria)
df_drop = df.select_and_drop(**criteria)
# indices
testing.assert_set_equal(df_select.index, expected)
testing.assert_set_equal(df_drop.index, expected)
# columns
testing.assert_set_equal(df_select.columns, df)
testing.assert_set_equal(df_drop.columns,
set(df_select.columns) - set(criteria))
# values
for i, c in itertools.product(df_select.index, df_select.columns):
assert df.loc[i, c] == f"{i}-{c}", "Erroneous values"
# instance
assert isinstance(df_select, utils.Selector)
assert isinstance(df_drop, utils.Selector)
def test_artificial_function_summary() -> None:
func = functionlib.ArtificialFunction("sphere", 5)
testing.assert_set_equal(func.descriptors.keys(), DESCRIPTION_KEYS)
np.testing.assert_equal(func.descriptors["function_class"],
"ArtificialFunction")