Example #1
0
def test_plot_optimization_history(direction: str) -> None:
    # Test with no studies.
    figure = plot_edf([])
    assert len(figure.get_lines()) == 0

    # Test with no trials.
    figure = plot_edf(create_study(direction=direction))
    assert len(figure.get_lines()) == 0

    figure = plot_edf(
        [create_study(direction=direction),
         create_study(direction=direction)])
    assert len(figure.get_lines()) == 0

    # Test with a study.
    study0 = create_study(direction=direction)
    study0.optimize(lambda t: t.suggest_float("x", 0, 5), n_trials=10)
    figure = plot_edf(study0)
    lines = figure.get_lines()
    _validate_edf_values(lines[0].get_ydata())
    assert len(lines) == 1
    assert figure.xaxis.label.get_text() == "Objective Value"

    # Test with two studies.
    study1 = create_study(direction=direction)
    study1.optimize(lambda t: t.suggest_float("x", 0, 5), n_trials=10)
    figure = plot_edf([study0, study1])
    lines = figure.get_lines()
    for line in lines:
        _validate_edf_values(line.get_ydata())
    assert len(lines) == 2

    figure = plot_edf((study0, study1))
    lines = figure.get_lines()
    for line in lines:
        _validate_edf_values(line.get_ydata())
    assert len(lines) == 2

    # Test with a customized target value.
    study0 = create_study(direction=direction)
    study0.optimize(lambda t: t.suggest_float("x", 0, 5), n_trials=10)
    with pytest.warns(UserWarning):
        figure = plot_edf(study0, target=lambda t: t.params["x"])
    lines = figure.get_lines()
    _validate_edf_values(lines[0].get_ydata())
    assert len(lines) == 1

    # Test with a customized target name.
    study0 = create_study(direction=direction)
    study0.optimize(lambda t: t.suggest_float("x", 0, 5), n_trials=10)
    figure = plot_edf(study0, target_name="Target Name")
    lines = figure.get_lines()
    _validate_edf_values(lines[0].get_ydata())
    assert len(figure.get_lines()) == 1
    assert figure.xaxis.label.get_text() == "Target Name"
Example #2
0
def _log_plots(run,
               study: optuna.Study,
               visualization_backend='plotly',
               log_plot_contour=True,
               log_plot_edf=True,
               log_plot_parallel_coordinate=True,
               log_plot_param_importances=True,
               log_plot_pareto_front=True,
               log_plot_slice=True,
               log_plot_intermediate_values=True,
               log_plot_optimization_history=True,
               ):
    if visualization_backend == 'matplotlib':
        import optuna.visualization.matplotlib as vis
    elif visualization_backend == 'plotly':
        import optuna.visualization as vis
    else:
        raise NotImplementedError(f'{visualization_backend} visualisation backend is not implemented')

    if vis.is_available:
        params = list(p_name for t in study.trials for p_name in t.params.keys())

        if log_plot_contour and any(params):
            run['visualizations/plot_contour'] = neptune.types.File.as_html(vis.plot_contour(study))

        if log_plot_edf:
            run['visualizations/plot_edf'] = neptune.types.File.as_html(vis.plot_edf(study))

        if log_plot_parallel_coordinate:
            run['visualizations/plot_parallel_coordinate'] = \
                neptune.types.File.as_html(vis.plot_parallel_coordinate(study))

        if log_plot_param_importances and len(study.get_trials(states=(optuna.trial.TrialState.COMPLETE, optuna.trial.TrialState.PRUNED,))) > 1:
            try:
                run['visualizations/plot_param_importances'] = neptune.types.File.as_html(vis.plot_param_importances(study))
            except (RuntimeError, ValueError, ZeroDivisionError):
                # Unable to compute importances
                pass

        if log_plot_pareto_front and study._is_multi_objective() and visualization_backend == 'plotly':
            run['visualizations/plot_pareto_front'] = neptune.types.File.as_html(vis.plot_pareto_front(study))

        if log_plot_slice and any(params):
            run['visualizations/plot_slice'] = neptune.types.File.as_html(vis.plot_slice(study))

        if log_plot_intermediate_values and any(trial.intermediate_values for trial in study.trials):
            # Intermediate values plot if available only if the above condition is met
            run['visualizations/plot_intermediate_values'] = \
                neptune.types.File.as_html(vis.plot_intermediate_values(study))

        if log_plot_optimization_history:
            run['visualizations/plot_optimization_history'] = \
                neptune.types.File.as_html(vis.plot_optimization_history(study))
Example #3
0
def test_plot_optimization_history(direction: str) -> None:
    # Test with no studies.
    figure = plot_edf([])
    assert not figure.has_data()

    # Test with no trials.
    figure = plot_edf(create_study(direction=direction))
    assert not figure.has_data()

    figure = plot_edf(
        [create_study(direction=direction),
         create_study(direction=direction)])
    assert not figure.has_data()

    # Test with a study.
    study0 = create_study(direction=direction)
    study0.optimize(lambda t: t.suggest_float("x", 0, 5), n_trials=10)
    figure = plot_edf(study0)
    assert figure.has_data()

    # Test with two studies.
    # TODO(ytknzw): Add more specific assertion with the numbers of the studies.
    study1 = create_study(direction=direction)
    study1.optimize(lambda t: t.suggest_float("x", 0, 5), n_trials=10)
    figure = plot_edf([study0, study1])
    assert figure.has_data()
    figure = plot_edf((study0, study1))
    assert figure.has_data()
Example #4
0
def test_inconsistent_number_of_trial_values() -> None:

    studies: List[Study] = []
    n_studies = 5

    for i in range(n_studies):
        study = prepare_study_with_trials()
        if i % 2 == 0:
            study.add_trial(create_trial(value=1.0))
        studies.append(study)

    figure = plot_edf(studies)
    assert len(figure.get_lines()) == n_studies
    plt.savefig(BytesIO())
Example #5
0
def test_plot_optimization_history(direction: str) -> None:
    # Test with no studies.
    figure = plot_edf([])
    assert not figure.has_data()

    # Test with no trials.
    figure = plot_edf(create_study(direction=direction))
    assert not figure.has_data()

    figure = plot_edf(
        [create_study(direction=direction),
         create_study(direction=direction)])
    assert not figure.has_data()

    # Test with a study.
    study0 = create_study(direction=direction)
    study0.optimize(lambda t: t.suggest_float("x", 0, 5), n_trials=10)
    figure = plot_edf(study0)
    assert figure.has_data()

    # Test with two studies.
    # TODO(ytknzw): Add more specific assertion with the numbers of the studies.
    study1 = create_study(direction=direction)
    study1.optimize(lambda t: t.suggest_float("x", 0, 5), n_trials=10)
    figure = plot_edf([study0, study1])
    assert figure.has_data()
    figure = plot_edf((study0, study1))
    assert figure.has_data()

    # Test with a customized target value.
    study0 = create_study(direction=direction)
    study0.optimize(lambda t: t.suggest_float("x", 0, 5), n_trials=10)
    with pytest.warns(UserWarning):
        figure = plot_edf(study0, target=lambda t: t.params["x"])
    assert figure.has_data()

    # Test with a customized target name.
    study0 = create_study(direction=direction)
    study0.optimize(lambda t: t.suggest_float("x", 0, 5), n_trials=10)
    figure = plot_edf(study0, target_name="Target Name")
    assert figure.has_data()
Example #6
0
def test_target_is_none_and_study_is_multi_obj() -> None:

    study = create_study(directions=["minimize", "minimize"])
    with pytest.raises(ValueError):
        plot_edf(study)
Example #7
0
def test_nonfinite_removed(value: int) -> None:

    study = prepare_study_with_trials(value_for_first_trial=value)
    figure = plot_edf(study)
    assert all(np.isfinite(figure.get_lines()[0].get_xdata()))
    plt.savefig(BytesIO())
Example #8
0
def test_nonfinite_multiobjective(objective: int, value: int) -> None:

    study = prepare_study_with_trials(n_objectives=2, value_for_first_trial=value)
    figure = plot_edf(study, target=lambda t: t.values[objective])
    assert all(np.isfinite(figure.get_lines()[0].get_xdata()))
    plt.savefig(BytesIO())