def test_batch_mode_parameter(batch_mode: bool, expected: tp.List[str]) -> None:
func = ExampleFunction(dimension=1, number=3)
optim = test_base.LoggingOptimizer(3)
with patch.object(xpbase.OptimizerSettings, "instantiate", return_value=optim):
xp = xpbase.Experiment(func, optimizer="OnePlusOne", budget=10, num_workers=3, batch_mode=batch_mode)
xp._run_with_error()
testing.printed_assert_equal(optim.logs, expected)
def test_folder_instantiator(clean_copy: bool) -> None:
path = Path(__file__).parent / "examples"
ifolder = instantiate.FolderInstantiator(path, clean_copy=clean_copy)
testing.printed_assert_equal(ifolder.placeholders, _EXPECTED)
np.testing.assert_equal(len(ifolder.file_functions), 1)
with testing.skip_error_on_systems(OSError, systems=("Windows", )):
with ifolder.instantiate(value1=12, value2=110., string="") as tmp:
with (tmp / "script.py").open("r") as f:
lines = f.readlines()
np.testing.assert_equal(lines[10], "value2 = 110.0\n")
def test_instrumented_function_kwarg_order() -> None:
ifunc = base.ExperimentFunction(_arg_return, p.Instrumentation( # type: ignore
kw4=p.Choice([1, 0]), kw2="constant", kw3=p.Array(shape=(2, 2)), kw1=p.Scalar(2.0).set_mutation(sigma=2.0)
))
np.testing.assert_equal(ifunc.dimension, 7)
data = np.array([-1, 1, 2, 3, 4, 100, -100])
args0, kwargs0 = ifunc.parametrization.spawn_child().set_standardized_data(data).value
# this is very stupid and should be removed when Parameter is in use
kwargs: tp.Any = ifunc(*args0, **kwargs0)[1] # type: ignore
testing.printed_assert_equal(kwargs, {"kw1": 0, "kw2": "constant", "kw3": [[1, 2], [3, 4]], "kw4": 1})
def test_get_nash() -> None:
zeroptim = xpvariants.Zero(parametrization=1, budget=4, num_workers=1)
param = zeroptim.parametrization
for k in range(4):
array = (float(k), )
zeroptim.archive[array] = utils.MultiValue(param, k, reference=param)
zeroptim.archive[array].count += (4 - k)
nash = utils._get_nash(zeroptim)
testing.printed_assert_equal(nash, [((2, ), 3), ((1, ), 4), ((0, ), 5)])
np.random.seed(12)
output = utils.sample_nash(zeroptim)
np.testing.assert_equal(output, (2, ))
def test_batch_and_steady_optimization(
num_workers: int, batch_mode: bool, expected: tp.List[tp.Tuple[str, float]]
) -> None:
# tests the suggestion (s) and update (u) patterns
# the w3_steady is unexpected. It is designed to be efficient with a non-sequential executor, but because
# of it, it is acting like batch mode when sequential...
optim = LoggingOptimizer(num_workers=num_workers)
func = CounterFunction()
with warnings.catch_warnings():
warnings.simplefilter("ignore")
optim.minimize(func, verbosity=2, batch_mode=batch_mode)
testing.printed_assert_equal(optim.logs, expected)
def test_instrumentation() -> None:
instru = p.Instrumentation(p.Scalar(), 3, b=p.Choice([0, 1, 2, 3]), a=p.TransitionChoice([0, 1, 2, 3]))
np.testing.assert_equal(instru.dimension, 6)
instru2 = p.Instrumentation(p.Scalar(), 3, b=p.Choice([0, 1, 2, 3]), a=p.TransitionChoice([0, 1, 2, 3]))
np.testing.assert_equal(instru2.dimension, 6)
data = instru2.spawn_child(new_value=((4, 3), dict(a=0, b=3))).get_standardized_data(reference=instru2)
np.testing.assert_array_almost_equal(data, [4, -1.1503, 0, 0, 0, 0.5878], decimal=4)
args, kwargs = instru.spawn_child().set_standardized_data(data, deterministic=True).value
testing.printed_assert_equal((args, kwargs), ((4.0, 3), {"a": 0, "b": 3}))
assert "3),Dict(a=TransitionChoice(choices=Tuple(0,1,2,3)," in repr(instru), f"Erroneous representation {instru}"
# check deterministic
data = np.array([0.0, 0, 0, 0, 0, 0])
total = 0
for _ in range(24):
total += instru.spawn_child().set_standardized_data(data, deterministic=True).kwargs["b"]
np.testing.assert_equal(total, 0)
# check stochastic
for _ in range(24):
total += instru.spawn_child().set_standardized_data(data, deterministic=False).kwargs["b"]
assert total != 0
# check duplicate
# instru2 = mvar.Instrumentation(*instru.args, **instru.kwargs) # TODO: OUCH SILENT FAIL
instru2.copy()
data = np.random.normal(0, 1, size=6)
testing.printed_assert_equal(instru2.spawn_child().set_standardized_data(data, deterministic=True).value,
instru.spawn_child().set_standardized_data(data, deterministic=True).value)
# check naming
instru_str = ("Instrumentation(Tuple(Scalar[sigma=Log{exp=2.0}],3),"
"Dict(a=TransitionChoice(choices=Tuple(0,1,2,3),"
"positions=Array{Cd(0,4)},transitions=[1. 1.]),"
"b=Choice(choices=Tuple(0,1,2,3),weights=Array{(1,4)})))")
testing.printed_assert_equal(instru.name, instru_str)
testing.printed_assert_equal("blublu", instru.set_name("blublu").name)
def test_deterministic_data_setter() -> None:
instru = p.Instrumentation(p.Choice([0, 1, 2, 3]), y=p.Choice([0, 1, 2, 3]))
ifunc = base.ExperimentFunction(_Callable(), instru)
data = [0.01, 0, 0, 0, 0.01, 0, 0, 0]
for _ in range(20):
args, kwargs = ifunc.parametrization.spawn_child().set_standardized_data(data, deterministic=True).value
testing.printed_assert_equal(args, [0])
testing.printed_assert_equal(kwargs, {"y": 0})
arg_sum, kwarg_sum = 0, 0
for _ in range(24):
args, kwargs = ifunc.parametrization.spawn_child().set_standardized_data(data, deterministic=False).value
arg_sum += args[0]
kwarg_sum += kwargs["y"]
assert arg_sum != 0
assert kwarg_sum != 0
def check_maker(maker: tp.Callable[[], tp.Iterator[experiments.Experiment]]) -> None:
generators = [maker() for _ in range(2)]
# check 1 sample
sample = next(maker())
assert isinstance(sample, experiments.Experiment)
# check names, coherence and non-randomness
for k, (elem1, elem2) in enumerate(itertools.zip_longest(*generators)):
assert not elem1.is_incoherent, f"Incoherent settings should be filtered out from generator:\n{elem1}"
try:
assert elem1 == elem2 # much faster but lacks explicit message
except AssertionError:
testing.printed_assert_equal(
elem1.get_description(),
elem2.get_description(),
err_msg=f"Two paths on the generator differed (see element #{k})\n"
"Generators need to be deterministic in order to split the workload!",
)
def test_deterministic_data_to_arguments() -> None:
instru = Instrumentation(var.SoftmaxCategorical([0, 1, 2, 3]),
y=var.SoftmaxCategorical([0, 1, 2, 3]))
ifunc = base.ExperimentFunction(_Callable(), instru)
data = [0.01, 0, 0, 0, 0.01, 0, 0, 0]
for _ in range(20):
args, kwargs = ifunc.parametrization.data_to_arguments(
data, deterministic=True)
testing.printed_assert_equal(args, [0])
testing.printed_assert_equal(kwargs, {"y": 0})
arg_sum, kwarg_sum = 0, 0
for _ in range(24):
args, kwargs = ifunc.parametrization.data_to_arguments(
data, deterministic=False)
arg_sum += args[0]
kwarg_sum += kwargs["y"]
assert arg_sum != 0
assert kwarg_sum != 0
def test_instrumented_function_kwarg_order() -> None:
ifunc = base.ExperimentFunction(
_arg_return,
Instrumentation( # type: ignore
kw4=var.SoftmaxCategorical([1, 0]),
kw2="constant",
kw3=var.Array(2, 2),
kw1=var.Gaussian(2, 2)))
np.testing.assert_equal(ifunc.dimension, 7)
data = np.array([-1, 1, 2, 3, 4, 100, -100])
args0, kwargs0 = ifunc.parametrization.data_to_arguments(data)
# this is very stupid and should be removed when Parameter is in use
kwargs: tp.Any = ifunc(*args0, **kwargs0)[1] # type: ignore
testing.printed_assert_equal(kwargs, {
"kw1": 0,
"kw2": "constant",
"kw3": [[1, 2], [3, 4]],
"kw4": 1
})
def test_experimented_function() -> None:
ifunc = base.ExperimentFunction(
_arg_return,
Instrumentation( # type: ignore
var.SoftmaxCategorical([1, 12]),
"constant",
var.Gaussian(0, 1, [2, 2]),
constkwarg="blublu",
plop=var.SoftmaxCategorical([3, 4]),
))
np.testing.assert_equal(ifunc.dimension, 8)
data = [-100.0, 100, 1, 2, 3, 4, 100, -100]
args0, kwargs0 = ifunc.parametrization.data_to_arguments(np.array(data))
output = ifunc(
*args0, **kwargs0
) # this is very stupid and should be removed when Parameter is in use
args: tp.Any = output[0] # type: ignore
kwargs: tp.Any = output[1] # type: ignore
testing.printed_assert_equal(args, [12, "constant", [[1, 2], [3, 4]]])
testing.printed_assert_equal(kwargs, {"constkwarg": "blublu", "plop": 3})
testing.printed_assert_equal(
ifunc.descriptors,
{
"dimension":
8,
"name":
"_arg_return",
"function_class":
"ExperimentFunction",
"instrumentation":
"SC(1,12|0),constant,G(0,1),constkwarg=blublu,plop=SC(3,4|0)",
},
)
def test_experiment_function() -> None:
param = ng.p.Instrumentation(
ng.p.Choice([1, 12]),
"constant",
ng.p.Array(shape=(2, 2)),
constkwarg="blublu",
plop=ng.p.Choice([3, 4]),
)
with pytest.raises(RuntimeError):
base.ExperimentFunction(_arg_return, param)
param.set_name("myparam")
ifunc = base.ExperimentFunction(_arg_return, param)
np.testing.assert_equal(ifunc.dimension, 8)
data = [-100.0, 100, 1, 2, 3, 4, 100, -100]
args0, kwargs0 = ifunc.parametrization.spawn_child().set_standardized_data(
data).value
output: tp.Any = ifunc(*args0, **kwargs0)
args: tp.Any = output[0]
kwargs: tp.Any = output[1]
testing.printed_assert_equal(args, [12, "constant", [[1, 2], [3, 4]]])
testing.printed_assert_equal(kwargs, {"constkwarg": "blublu", "plop": 3})
testing.printed_assert_equal(
ifunc.descriptors,
{
"dimension": 8,
"name": "_arg_return",
"function_class": "ExperimentFunction",
"parametrization": "myparam",
},
)
def test_experiment_function() -> None:
ifunc = base.ExperimentFunction(
_arg_return,
p.Instrumentation( # type: ignore
p.Choice([1, 12]),
"constant",
p.Array(shape=(2, 2)),
constkwarg="blublu",
plop=p.Choice([3, 4]),
))
np.testing.assert_equal(ifunc.dimension, 8)
data = [-100.0, 100, 1, 2, 3, 4, 100, -100]
args0, kwargs0 = ifunc.parametrization.spawn_child().set_standardized_data(
data).value
output = ifunc(
*args0, **kwargs0
) # this is very stupid and should be removed when Parameter is in use
args: tp.Any = output[0] # type: ignore
kwargs: tp.Any = output[1] # type: ignore
testing.printed_assert_equal(args, [12, "constant", [[1, 2], [3, 4]]])
testing.printed_assert_equal(kwargs, {"constkwarg": "blublu", "plop": 3})
instru_str = ("Instrumentation(Tuple(Choice(choices=Tuple(1,12),"
"weights=Array{(1,2)}),constant,"
"Array{(2,2)}),"
"Dict(constkwarg=blublu,plop=Choice(choices=Tuple(3,4),"
"weights=Array{(1,2)})))")
testing.printed_assert_equal(
ifunc.descriptors,
{
"dimension": 8,
"name": "_arg_return",
"function_class": "ExperimentFunction",
"parametrization": instru_str,
},
)
def test_experimented_function() -> None:
ifunc = base.ExperimentFunction(
_arg_return,
Instrumentation( # type: ignore
var.SoftmaxCategorical([1, 12]),
"constant",
var.Gaussian(0, 1, [2, 2]),
constkwarg="blublu",
plop=var.SoftmaxCategorical([3, 4]),
))
np.testing.assert_equal(ifunc.dimension, 8)
data = [-100.0, 100, 1, 2, 3, 4, 100, -100]
args0, kwargs0 = ifunc.parametrization.data_to_arguments(np.array(data))
output = ifunc(
*args0, **kwargs0
) # this is very stupid and should be removed when Parameter is in use
args: tp.Any = output[0] # type: ignore
kwargs: tp.Any = output[1] # type: ignore
testing.printed_assert_equal(args, [12, "constant", [[1, 2], [3, 4]]])
testing.printed_assert_equal(kwargs, {"constkwarg": "blublu", "plop": 3})
instru_str = (
"Instrumentation(Tuple(SoftmaxCategorical(choices=Tuple(1,12),"
"weights=Array{(2,)}[recombination=average,sigma=1.0]),constant,G(0,1)),"
"Dict(constkwarg=blublu,plop=SoftmaxCategorical(choices=Tuple(3,4),"
"weights=Array{(2,)}[recombination=average,sigma=1.0])))")
testing.printed_assert_equal(
ifunc.descriptors,
{
"dimension": 8,
"name": "_arg_return",
"function_class": "ExperimentFunction",
"instrumentation": instru_str,
},
)
def test_placeholder(
text: str, name_comments: tp.List[tp.Tuple[str,
tp.Optional[str]]]) -> None:
placeholders = Placeholder.finditer(text)
testing.printed_assert_equal(placeholders,
[Placeholder(*x) for x in name_comments])
def test_file_text_function() -> None:
path = Path(__file__).parent / "examples" / "script.py"
filefunc = instantiate.FileTextFunction(path)
testing.printed_assert_equal(filefunc.placeholders, _EXPECTED)
def test_selector_unique(columns: tp.List[str],
expected: tp.Iterable[tp.Tuple[int, int]]) -> None:
df = utils.Selector(index=["i1", "i2", "i3"],
columns=["c1", "c2"],
data=[[2, 1], [2, 2], [2, 1]])
testing.printed_assert_equal(df.unique(columns), expected)
