Beispiel #1
0
def test_experimented_function() -> None:
    ifunc = base.ExperimentFunction(
        _arg_return,
        Instrumentation(  # type: ignore
            var.SoftmaxCategorical([1, 12]),
            "constant",
            var.Gaussian(0, 1, [2, 2]),
            constkwarg="blublu",
            plop=var.SoftmaxCategorical([3, 4]),
        ))
    np.testing.assert_equal(ifunc.dimension, 8)
    data = [-100.0, 100, 1, 2, 3, 4, 100, -100]
    args0, kwargs0 = ifunc.parametrization.data_to_arguments(np.array(data))
    output = ifunc(
        *args0, **kwargs0
    )  # this is very stupid and should be removed when Parameter is in use
    args: tp.Any = output[0]  # type: ignore
    kwargs: tp.Any = output[1]  # type: ignore
    testing.printed_assert_equal(args, [12, "constant", [[1, 2], [3, 4]]])
    testing.printed_assert_equal(kwargs, {"constkwarg": "blublu", "plop": 3})
    instru_str = (
        "Instrumentation(Tuple(SoftmaxCategorical(choices=Tuple(1,12),"
        "weights=Array{(2,)}[recombination=average,sigma=1.0]),constant,G(0,1)),"
        "Dict(constkwarg=blublu,plop=SoftmaxCategorical(choices=Tuple(3,4),"
        "weights=Array{(2,)}[recombination=average,sigma=1.0])))")
    testing.printed_assert_equal(
        ifunc.descriptors,
        {
            "dimension": 8,
            "name": "_arg_return",
            "function_class": "ExperimentFunction",
            "instrumentation": instru_str,
        },
    )
Beispiel #2
0
def test_instrumentation_continuous_noisy(variables: tp.Tuple[var.Variable,
                                                              ...],
                                          continuous: bool,
                                          noisy: bool) -> None:
    instru = Instrumentation(*variables)
    assert instru.continuous == continuous
    assert instru.noisy == noisy
Beispiel #3
0
def test_experimented_function() -> None:
    ifunc = base.ExperimentFunction(
        _arg_return,
        Instrumentation(  # type: ignore
            var.SoftmaxCategorical([1, 12]),
            "constant",
            var.Gaussian(0, 1, [2, 2]),
            constkwarg="blublu",
            plop=var.SoftmaxCategorical([3, 4]),
        ))
    np.testing.assert_equal(ifunc.dimension, 8)
    data = [-100.0, 100, 1, 2, 3, 4, 100, -100]
    args0, kwargs0 = ifunc.parametrization.data_to_arguments(np.array(data))
    output = ifunc(
        *args0, **kwargs0
    )  # this is very stupid and should be removed when Parameter is in use
    args: tp.Any = output[0]  # type: ignore
    kwargs: tp.Any = output[1]  # type: ignore
    testing.printed_assert_equal(args, [12, "constant", [[1, 2], [3, 4]]])
    testing.printed_assert_equal(kwargs, {"constkwarg": "blublu", "plop": 3})
    testing.printed_assert_equal(
        ifunc.descriptors,
        {
            "dimension":
            8,
            "name":
            "_arg_return",
            "function_class":
            "ExperimentFunction",
            "instrumentation":
            "SC(1,12|0),constant,G(0,1),constkwarg=blublu,plop=SC(3,4|0)",
        },
    )
Beispiel #4
0
def find_best_parametrization(exp_dir,
                              metrics_coefs,
                              preprocessors_kwargs,
                              parametrization_budget=64):
    @lru_cache()
    def evaluate_parametrization(**instru_kwargs):
        # Note that we use default generate kwargs instead of provided one because they are faster
        preprocessors_kwargs = instru_kwargs_to_preprocessors_kwargs(
            instru_kwargs)
        simplifier = get_simplifier(exp_dir,
                                    preprocessors_kwargs=preprocessors_kwargs,
                                    generate_kwargs={})
        scores = evaluate_simplifier_on_turkcorpus(simplifier, phase='valid')
        return combine_metrics(scores['BLEU'], scores['SARI'], scores['FKGL'],
                               metrics_coefs)

    def preprocessors_kwargs_to_instru_kwargs(preprocessors_kwargs):
        instru_kwargs = {}
        for preprocessor_name, preprocessor_kwargs in preprocessors_kwargs.items(
        ):
            assert '_' not in preprocessor_name
            preprocessor = get_preprocessor_by_name(preprocessor_name)(
                **preprocessor_kwargs)
            # First we set the values from preprocessors_kwargs which are constant
            for kwarg_name, kwarg_value in preprocessor_kwargs.items():
                instru_kwargs[
                    f'{preprocessor_name}_{kwarg_name}'] = kwarg_value
            # Then we overwrite some of these values with nevergrad variables when necessary
            for kwarg_name, kwarg_value in preprocessor.get_nevergrad_variables(
            ).items():
                instru_kwargs[
                    f'{preprocessor_name}_{kwarg_name}'] = kwarg_value
        return instru_kwargs

    def instru_kwargs_to_preprocessors_kwargs(instru_kwargs):
        preprocessors_kwargs = defaultdict(dict)
        for key, value in instru_kwargs.items():
            preprocessor_name, kwarg_name = re.match(
                r'([a-zA-Z0-9]+)_([a-z0-9_]+)', key).groups()
            preprocessors_kwargs[preprocessor_name][kwarg_name] = value
        return dict(preprocessors_kwargs)

    instru_kwargs = preprocessors_kwargs_to_instru_kwargs(preprocessors_kwargs)
    instru = Instrumentation(**instru_kwargs)
    if instru.dimension == 0:
        return preprocessors_kwargs
    # No need to search a lot when there is only a few parameters
    parametrization_budget = min(32**instru.dimension, parametrization_budget)
    optimizer = optimizerlib.ScrHammersleySearch(instrumentation=instru,
                                                 budget=parametrization_budget,
                                                 num_workers=1)
    recommendation = optimizer.optimize(evaluate_parametrization, verbosity=0)
    return instru_kwargs_to_preprocessors_kwargs(recommendation.kwargs)
Beispiel #5
0
def test_deterministic_data_to_arguments() -> None:
    instru = Instrumentation(var.SoftmaxCategorical([0, 1, 2, 3]),
                             y=var.SoftmaxCategorical([0, 1, 2, 3]))
    ifunc = base.ExperimentFunction(_Callable(), instru)
    data = [0.01, 0, 0, 0, 0.01, 0, 0, 0]
    for _ in range(20):
        args, kwargs = ifunc.parametrization.data_to_arguments(
            data, deterministic=True)
        testing.printed_assert_equal(args, [0])
        testing.printed_assert_equal(kwargs, {"y": 0})
    arg_sum, kwarg_sum = 0, 0
    for _ in range(24):
        args, kwargs = ifunc.parametrization.data_to_arguments(
            data, deterministic=False)
        arg_sum += args[0]
        kwarg_sum += kwargs["y"]
    assert arg_sum != 0
    assert kwarg_sum != 0
Beispiel #6
0
def test_instrumented_function_kwarg_order() -> None:
    ifunc = base.ExperimentFunction(
        _arg_return,
        Instrumentation(  # type: ignore
            kw4=var.SoftmaxCategorical([1, 0]),
            kw2="constant",
            kw3=var.Array(2, 2),
            kw1=var.Gaussian(2, 2)))
    np.testing.assert_equal(ifunc.dimension, 7)
    data = np.array([-1, 1, 2, 3, 4, 100, -100])
    args0, kwargs0 = ifunc.parametrization.data_to_arguments(data)
    # this is very stupid and should be removed when Parameter is in use
    kwargs: tp.Any = ifunc(*args0, **kwargs0)[1]  # type: ignore
    testing.printed_assert_equal(kwargs, {
        "kw1": 0,
        "kw2": "constant",
        "kw3": [[1, 2], [3, 4]],
        "kw4": 1
    })