def test_experimented_function() -> None: ifunc = base.ExperimentFunction( _arg_return, Instrumentation( # type: ignore var.SoftmaxCategorical([1, 12]), "constant", var.Gaussian(0, 1, [2, 2]), constkwarg="blublu", plop=var.SoftmaxCategorical([3, 4]), )) np.testing.assert_equal(ifunc.dimension, 8) data = [-100.0, 100, 1, 2, 3, 4, 100, -100] args0, kwargs0 = ifunc.parametrization.data_to_arguments(np.array(data)) output = ifunc( *args0, **kwargs0 ) # this is very stupid and should be removed when Parameter is in use args: tp.Any = output[0] # type: ignore kwargs: tp.Any = output[1] # type: ignore testing.printed_assert_equal(args, [12, "constant", [[1, 2], [3, 4]]]) testing.printed_assert_equal(kwargs, {"constkwarg": "blublu", "plop": 3}) instru_str = ( "Instrumentation(Tuple(SoftmaxCategorical(choices=Tuple(1,12)," "weights=Array{(2,)}[recombination=average,sigma=1.0]),constant,G(0,1))," "Dict(constkwarg=blublu,plop=SoftmaxCategorical(choices=Tuple(3,4)," "weights=Array{(2,)}[recombination=average,sigma=1.0])))") testing.printed_assert_equal( ifunc.descriptors, { "dimension": 8, "name": "_arg_return", "function_class": "ExperimentFunction", "instrumentation": instru_str, }, )
def test_instrumentation_continuous_noisy(variables: tp.Tuple[var.Variable, ...], continuous: bool, noisy: bool) -> None: instru = Instrumentation(*variables) assert instru.continuous == continuous assert instru.noisy == noisy
def test_experimented_function() -> None: ifunc = base.ExperimentFunction( _arg_return, Instrumentation( # type: ignore var.SoftmaxCategorical([1, 12]), "constant", var.Gaussian(0, 1, [2, 2]), constkwarg="blublu", plop=var.SoftmaxCategorical([3, 4]), )) np.testing.assert_equal(ifunc.dimension, 8) data = [-100.0, 100, 1, 2, 3, 4, 100, -100] args0, kwargs0 = ifunc.parametrization.data_to_arguments(np.array(data)) output = ifunc( *args0, **kwargs0 ) # this is very stupid and should be removed when Parameter is in use args: tp.Any = output[0] # type: ignore kwargs: tp.Any = output[1] # type: ignore testing.printed_assert_equal(args, [12, "constant", [[1, 2], [3, 4]]]) testing.printed_assert_equal(kwargs, {"constkwarg": "blublu", "plop": 3}) testing.printed_assert_equal( ifunc.descriptors, { "dimension": 8, "name": "_arg_return", "function_class": "ExperimentFunction", "instrumentation": "SC(1,12|0),constant,G(0,1),constkwarg=blublu,plop=SC(3,4|0)", }, )
def find_best_parametrization(exp_dir, metrics_coefs, preprocessors_kwargs, parametrization_budget=64): @lru_cache() def evaluate_parametrization(**instru_kwargs): # Note that we use default generate kwargs instead of provided one because they are faster preprocessors_kwargs = instru_kwargs_to_preprocessors_kwargs( instru_kwargs) simplifier = get_simplifier(exp_dir, preprocessors_kwargs=preprocessors_kwargs, generate_kwargs={}) scores = evaluate_simplifier_on_turkcorpus(simplifier, phase='valid') return combine_metrics(scores['BLEU'], scores['SARI'], scores['FKGL'], metrics_coefs) def preprocessors_kwargs_to_instru_kwargs(preprocessors_kwargs): instru_kwargs = {} for preprocessor_name, preprocessor_kwargs in preprocessors_kwargs.items( ): assert '_' not in preprocessor_name preprocessor = get_preprocessor_by_name(preprocessor_name)( **preprocessor_kwargs) # First we set the values from preprocessors_kwargs which are constant for kwarg_name, kwarg_value in preprocessor_kwargs.items(): instru_kwargs[ f'{preprocessor_name}_{kwarg_name}'] = kwarg_value # Then we overwrite some of these values with nevergrad variables when necessary for kwarg_name, kwarg_value in preprocessor.get_nevergrad_variables( ).items(): instru_kwargs[ f'{preprocessor_name}_{kwarg_name}'] = kwarg_value return instru_kwargs def instru_kwargs_to_preprocessors_kwargs(instru_kwargs): preprocessors_kwargs = defaultdict(dict) for key, value in instru_kwargs.items(): preprocessor_name, kwarg_name = re.match( r'([a-zA-Z0-9]+)_([a-z0-9_]+)', key).groups() preprocessors_kwargs[preprocessor_name][kwarg_name] = value return dict(preprocessors_kwargs) instru_kwargs = preprocessors_kwargs_to_instru_kwargs(preprocessors_kwargs) instru = Instrumentation(**instru_kwargs) if instru.dimension == 0: return preprocessors_kwargs # No need to search a lot when there is only a few parameters parametrization_budget = min(32**instru.dimension, parametrization_budget) optimizer = optimizerlib.ScrHammersleySearch(instrumentation=instru, budget=parametrization_budget, num_workers=1) recommendation = optimizer.optimize(evaluate_parametrization, verbosity=0) return instru_kwargs_to_preprocessors_kwargs(recommendation.kwargs)
def test_deterministic_data_to_arguments() -> None: instru = Instrumentation(var.SoftmaxCategorical([0, 1, 2, 3]), y=var.SoftmaxCategorical([0, 1, 2, 3])) ifunc = base.ExperimentFunction(_Callable(), instru) data = [0.01, 0, 0, 0, 0.01, 0, 0, 0] for _ in range(20): args, kwargs = ifunc.parametrization.data_to_arguments( data, deterministic=True) testing.printed_assert_equal(args, [0]) testing.printed_assert_equal(kwargs, {"y": 0}) arg_sum, kwarg_sum = 0, 0 for _ in range(24): args, kwargs = ifunc.parametrization.data_to_arguments( data, deterministic=False) arg_sum += args[0] kwarg_sum += kwargs["y"] assert arg_sum != 0 assert kwarg_sum != 0
def test_instrumented_function_kwarg_order() -> None: ifunc = base.ExperimentFunction( _arg_return, Instrumentation( # type: ignore kw4=var.SoftmaxCategorical([1, 0]), kw2="constant", kw3=var.Array(2, 2), kw1=var.Gaussian(2, 2))) np.testing.assert_equal(ifunc.dimension, 7) data = np.array([-1, 1, 2, 3, 4, 100, -100]) args0, kwargs0 = ifunc.parametrization.data_to_arguments(data) # this is very stupid and should be removed when Parameter is in use kwargs: tp.Any = ifunc(*args0, **kwargs0)[1] # type: ignore testing.printed_assert_equal(kwargs, { "kw1": 0, "kw2": "constant", "kw3": [[1, 2], [3, 4]], "kw4": 1 })