def test_batch_mode_parameter(batch_mode: bool, expected: tp.List[str]) -> None: func = ExampleFunction(dimension=1, number=3) optim = test_base.LoggingOptimizer(3) with patch.object(xpbase.OptimizerSettings, "instantiate", return_value=optim): xp = xpbase.Experiment(func, optimizer="OnePlusOne", budget=10, num_workers=3, batch_mode=batch_mode) xp._run_with_error() testing.printed_assert_equal(optim.logs, expected)
def test_folder_instantiator(clean_copy: bool) -> None: path = Path(__file__).parent / "examples" ifolder = instantiate.FolderInstantiator(path, clean_copy=clean_copy) testing.printed_assert_equal(ifolder.placeholders, _EXPECTED) np.testing.assert_equal(len(ifolder.file_functions), 1) with testing.skip_error_on_systems(OSError, systems=("Windows", )): with ifolder.instantiate(value1=12, value2=110., string="") as tmp: with (tmp / "script.py").open("r") as f: lines = f.readlines() np.testing.assert_equal(lines[10], "value2 = 110.0\n")
def test_instrumented_function_kwarg_order() -> None: ifunc = base.ExperimentFunction(_arg_return, p.Instrumentation( # type: ignore kw4=p.Choice([1, 0]), kw2="constant", kw3=p.Array(shape=(2, 2)), kw1=p.Scalar(2.0).set_mutation(sigma=2.0) )) np.testing.assert_equal(ifunc.dimension, 7) data = np.array([-1, 1, 2, 3, 4, 100, -100]) args0, kwargs0 = ifunc.parametrization.spawn_child().set_standardized_data(data).value # this is very stupid and should be removed when Parameter is in use kwargs: tp.Any = ifunc(*args0, **kwargs0)[1] # type: ignore testing.printed_assert_equal(kwargs, {"kw1": 0, "kw2": "constant", "kw3": [[1, 2], [3, 4]], "kw4": 1})
def test_get_nash() -> None: zeroptim = xpvariants.Zero(parametrization=1, budget=4, num_workers=1) param = zeroptim.parametrization for k in range(4): array = (float(k), ) zeroptim.archive[array] = utils.MultiValue(param, k, reference=param) zeroptim.archive[array].count += (4 - k) nash = utils._get_nash(zeroptim) testing.printed_assert_equal(nash, [((2, ), 3), ((1, ), 4), ((0, ), 5)]) np.random.seed(12) output = utils.sample_nash(zeroptim) np.testing.assert_equal(output, (2, ))
def test_batch_and_steady_optimization( num_workers: int, batch_mode: bool, expected: tp.List[tp.Tuple[str, float]] ) -> None: # tests the suggestion (s) and update (u) patterns # the w3_steady is unexpected. It is designed to be efficient with a non-sequential executor, but because # of it, it is acting like batch mode when sequential... optim = LoggingOptimizer(num_workers=num_workers) func = CounterFunction() with warnings.catch_warnings(): warnings.simplefilter("ignore") optim.minimize(func, verbosity=2, batch_mode=batch_mode) testing.printed_assert_equal(optim.logs, expected)
def test_instrumentation() -> None: instru = p.Instrumentation(p.Scalar(), 3, b=p.Choice([0, 1, 2, 3]), a=p.TransitionChoice([0, 1, 2, 3])) np.testing.assert_equal(instru.dimension, 6) instru2 = p.Instrumentation(p.Scalar(), 3, b=p.Choice([0, 1, 2, 3]), a=p.TransitionChoice([0, 1, 2, 3])) np.testing.assert_equal(instru2.dimension, 6) data = instru2.spawn_child(new_value=((4, 3), dict(a=0, b=3))).get_standardized_data(reference=instru2) np.testing.assert_array_almost_equal(data, [4, -1.1503, 0, 0, 0, 0.5878], decimal=4) args, kwargs = instru.spawn_child().set_standardized_data(data, deterministic=True).value testing.printed_assert_equal((args, kwargs), ((4.0, 3), {"a": 0, "b": 3})) assert "3),Dict(a=TransitionChoice(choices=Tuple(0,1,2,3)," in repr(instru), f"Erroneous representation {instru}" # check deterministic data = np.array([0.0, 0, 0, 0, 0, 0]) total = 0 for _ in range(24): total += instru.spawn_child().set_standardized_data(data, deterministic=True).kwargs["b"] np.testing.assert_equal(total, 0) # check stochastic for _ in range(24): total += instru.spawn_child().set_standardized_data(data, deterministic=False).kwargs["b"] assert total != 0 # check duplicate # instru2 = mvar.Instrumentation(*instru.args, **instru.kwargs) # TODO: OUCH SILENT FAIL instru2.copy() data = np.random.normal(0, 1, size=6) testing.printed_assert_equal(instru2.spawn_child().set_standardized_data(data, deterministic=True).value, instru.spawn_child().set_standardized_data(data, deterministic=True).value) # check naming instru_str = ("Instrumentation(Tuple(Scalar[sigma=Log{exp=2.0}],3)," "Dict(a=TransitionChoice(choices=Tuple(0,1,2,3)," "positions=Array{Cd(0,4)},transitions=[1. 1.])," "b=Choice(choices=Tuple(0,1,2,3),weights=Array{(1,4)})))") testing.printed_assert_equal(instru.name, instru_str) testing.printed_assert_equal("blublu", instru.set_name("blublu").name)
def test_deterministic_data_setter() -> None: instru = p.Instrumentation(p.Choice([0, 1, 2, 3]), y=p.Choice([0, 1, 2, 3])) ifunc = base.ExperimentFunction(_Callable(), instru) data = [0.01, 0, 0, 0, 0.01, 0, 0, 0] for _ in range(20): args, kwargs = ifunc.parametrization.spawn_child().set_standardized_data(data, deterministic=True).value testing.printed_assert_equal(args, [0]) testing.printed_assert_equal(kwargs, {"y": 0}) arg_sum, kwarg_sum = 0, 0 for _ in range(24): args, kwargs = ifunc.parametrization.spawn_child().set_standardized_data(data, deterministic=False).value arg_sum += args[0] kwarg_sum += kwargs["y"] assert arg_sum != 0 assert kwarg_sum != 0
def check_maker(maker: tp.Callable[[], tp.Iterator[experiments.Experiment]]) -> None: generators = [maker() for _ in range(2)] # check 1 sample sample = next(maker()) assert isinstance(sample, experiments.Experiment) # check names, coherence and non-randomness for k, (elem1, elem2) in enumerate(itertools.zip_longest(*generators)): assert not elem1.is_incoherent, f"Incoherent settings should be filtered out from generator:\n{elem1}" try: assert elem1 == elem2 # much faster but lacks explicit message except AssertionError: testing.printed_assert_equal( elem1.get_description(), elem2.get_description(), err_msg=f"Two paths on the generator differed (see element #{k})\n" "Generators need to be deterministic in order to split the workload!", )
def test_deterministic_data_to_arguments() -> None: instru = Instrumentation(var.SoftmaxCategorical([0, 1, 2, 3]), y=var.SoftmaxCategorical([0, 1, 2, 3])) ifunc = base.ExperimentFunction(_Callable(), instru) data = [0.01, 0, 0, 0, 0.01, 0, 0, 0] for _ in range(20): args, kwargs = ifunc.parametrization.data_to_arguments( data, deterministic=True) testing.printed_assert_equal(args, [0]) testing.printed_assert_equal(kwargs, {"y": 0}) arg_sum, kwarg_sum = 0, 0 for _ in range(24): args, kwargs = ifunc.parametrization.data_to_arguments( data, deterministic=False) arg_sum += args[0] kwarg_sum += kwargs["y"] assert arg_sum != 0 assert kwarg_sum != 0
def test_instrumented_function_kwarg_order() -> None: ifunc = base.ExperimentFunction( _arg_return, Instrumentation( # type: ignore kw4=var.SoftmaxCategorical([1, 0]), kw2="constant", kw3=var.Array(2, 2), kw1=var.Gaussian(2, 2))) np.testing.assert_equal(ifunc.dimension, 7) data = np.array([-1, 1, 2, 3, 4, 100, -100]) args0, kwargs0 = ifunc.parametrization.data_to_arguments(data) # this is very stupid and should be removed when Parameter is in use kwargs: tp.Any = ifunc(*args0, **kwargs0)[1] # type: ignore testing.printed_assert_equal(kwargs, { "kw1": 0, "kw2": "constant", "kw3": [[1, 2], [3, 4]], "kw4": 1 })
def test_experimented_function() -> None: ifunc = base.ExperimentFunction( _arg_return, Instrumentation( # type: ignore var.SoftmaxCategorical([1, 12]), "constant", var.Gaussian(0, 1, [2, 2]), constkwarg="blublu", plop=var.SoftmaxCategorical([3, 4]), )) np.testing.assert_equal(ifunc.dimension, 8) data = [-100.0, 100, 1, 2, 3, 4, 100, -100] args0, kwargs0 = ifunc.parametrization.data_to_arguments(np.array(data)) output = ifunc( *args0, **kwargs0 ) # this is very stupid and should be removed when Parameter is in use args: tp.Any = output[0] # type: ignore kwargs: tp.Any = output[1] # type: ignore testing.printed_assert_equal(args, [12, "constant", [[1, 2], [3, 4]]]) testing.printed_assert_equal(kwargs, {"constkwarg": "blublu", "plop": 3}) testing.printed_assert_equal( ifunc.descriptors, { "dimension": 8, "name": "_arg_return", "function_class": "ExperimentFunction", "instrumentation": "SC(1,12|0),constant,G(0,1),constkwarg=blublu,plop=SC(3,4|0)", }, )
def test_experiment_function() -> None: param = ng.p.Instrumentation( ng.p.Choice([1, 12]), "constant", ng.p.Array(shape=(2, 2)), constkwarg="blublu", plop=ng.p.Choice([3, 4]), ) with pytest.raises(RuntimeError): base.ExperimentFunction(_arg_return, param) param.set_name("myparam") ifunc = base.ExperimentFunction(_arg_return, param) np.testing.assert_equal(ifunc.dimension, 8) data = [-100.0, 100, 1, 2, 3, 4, 100, -100] args0, kwargs0 = ifunc.parametrization.spawn_child().set_standardized_data( data).value output: tp.Any = ifunc(*args0, **kwargs0) args: tp.Any = output[0] kwargs: tp.Any = output[1] testing.printed_assert_equal(args, [12, "constant", [[1, 2], [3, 4]]]) testing.printed_assert_equal(kwargs, {"constkwarg": "blublu", "plop": 3}) testing.printed_assert_equal( ifunc.descriptors, { "dimension": 8, "name": "_arg_return", "function_class": "ExperimentFunction", "parametrization": "myparam", }, )
def test_experiment_function() -> None: ifunc = base.ExperimentFunction( _arg_return, p.Instrumentation( # type: ignore p.Choice([1, 12]), "constant", p.Array(shape=(2, 2)), constkwarg="blublu", plop=p.Choice([3, 4]), )) np.testing.assert_equal(ifunc.dimension, 8) data = [-100.0, 100, 1, 2, 3, 4, 100, -100] args0, kwargs0 = ifunc.parametrization.spawn_child().set_standardized_data( data).value output = ifunc( *args0, **kwargs0 ) # this is very stupid and should be removed when Parameter is in use args: tp.Any = output[0] # type: ignore kwargs: tp.Any = output[1] # type: ignore testing.printed_assert_equal(args, [12, "constant", [[1, 2], [3, 4]]]) testing.printed_assert_equal(kwargs, {"constkwarg": "blublu", "plop": 3}) instru_str = ("Instrumentation(Tuple(Choice(choices=Tuple(1,12)," "weights=Array{(1,2)}),constant," "Array{(2,2)})," "Dict(constkwarg=blublu,plop=Choice(choices=Tuple(3,4)," "weights=Array{(1,2)})))") testing.printed_assert_equal( ifunc.descriptors, { "dimension": 8, "name": "_arg_return", "function_class": "ExperimentFunction", "parametrization": instru_str, }, )
def test_experimented_function() -> None: ifunc = base.ExperimentFunction( _arg_return, Instrumentation( # type: ignore var.SoftmaxCategorical([1, 12]), "constant", var.Gaussian(0, 1, [2, 2]), constkwarg="blublu", plop=var.SoftmaxCategorical([3, 4]), )) np.testing.assert_equal(ifunc.dimension, 8) data = [-100.0, 100, 1, 2, 3, 4, 100, -100] args0, kwargs0 = ifunc.parametrization.data_to_arguments(np.array(data)) output = ifunc( *args0, **kwargs0 ) # this is very stupid and should be removed when Parameter is in use args: tp.Any = output[0] # type: ignore kwargs: tp.Any = output[1] # type: ignore testing.printed_assert_equal(args, [12, "constant", [[1, 2], [3, 4]]]) testing.printed_assert_equal(kwargs, {"constkwarg": "blublu", "plop": 3}) instru_str = ( "Instrumentation(Tuple(SoftmaxCategorical(choices=Tuple(1,12)," "weights=Array{(2,)}[recombination=average,sigma=1.0]),constant,G(0,1))," "Dict(constkwarg=blublu,plop=SoftmaxCategorical(choices=Tuple(3,4)," "weights=Array{(2,)}[recombination=average,sigma=1.0])))") testing.printed_assert_equal( ifunc.descriptors, { "dimension": 8, "name": "_arg_return", "function_class": "ExperimentFunction", "instrumentation": instru_str, }, )
def test_placeholder( text: str, name_comments: tp.List[tp.Tuple[str, tp.Optional[str]]]) -> None: placeholders = Placeholder.finditer(text) testing.printed_assert_equal(placeholders, [Placeholder(*x) for x in name_comments])
def test_file_text_function() -> None: path = Path(__file__).parent / "examples" / "script.py" filefunc = instantiate.FileTextFunction(path) testing.printed_assert_equal(filefunc.placeholders, _EXPECTED)
def test_selector_unique(columns: tp.List[str], expected: tp.Iterable[tp.Tuple[int, int]]) -> None: df = utils.Selector(index=["i1", "i2", "i3"], columns=["c1", "c2"], data=[[2, 1], [2, 2], [2, 1]]) testing.printed_assert_equal(df.unique(columns), expected)