def test_custom_n_samples(self, monkeypatch): """Test changing n_samples value""" mock_train_regressor(monkeypatch) config = mock_space() mock_experiment(monkeypatch) PARAMS = ["x", "y"] N_SAMPLES = numpy.random.randint(20, 50) def mock_partial_dependency_grid(space, model, params, samples, n_points): print(samples) assert samples.shape == (N_SAMPLES, len(PARAMS)) return partial_dependency_grid(space, model, params, samples, n_points) monkeypatch.setattr( "orion.analysis.partial_dependency_utils.partial_dependency_grid", mock_partial_dependency_grid, ) with create_experiment(config, trial_config) as (_, _, experiment): plot = partial_dependencies( experiment, n_grid_points=10, model_kwargs=dict(random_state=1), n_samples=N_SAMPLES, )
def test_categorical_multidim(self, monkeypatch): """Tests that multidim categorical is supported""" mock_train_regressor(monkeypatch) config = mock_space(y='choices(["a", "b", "c"], shape=3)') mock_experiment( monkeypatch, y=[["c", "b", "a"], ["c", "a", "c"], ["a", "b", "a"], ["c", "b", "b"]], ) with create_experiment(config, trial_config) as (_, _, experiment): plot = partial_dependencies(experiment, n_grid_points=5, model_kwargs=dict(random_state=1)) assert_partial_dependencies_plot( plot, dims=["x", "y[0]", "y[1]", "y[2]"], n_grid_points={ "x": 5, "y[0]": 3, "y[1]": 3, "y[2]": 3 }, )
def test_ignore_uncompleted_statuses(self, monkeypatch): """Tests that uncompleted statuses are ignored""" mock_train_regressor(monkeypatch) config = mock_space() mock_experiment( monkeypatch, ids="abcdefgh", x=[0, 0, 0, 1, 0, 2, 0, 3], y=[1, 0, 0, 2, 0, 0, 0, 3], objectives=[0.1, None, None, 0.2, None, 0.3, None, 0.5], status=[ "completed", "new", "reserved", "completed", "broken", "completed", "interrupted", "completed", ], ) with create_experiment(config, trial_config) as (_, _, experiment): plot = partial_dependencies(experiment, n_grid_points=5) assert_partial_dependencies_plot(plot, dims=["x", "y"])
def test_fidelity(self, monkeypatch): """Tests that fidelity is supported""" mock_train_regressor(monkeypatch) config = mock_space(y="fidelity(1, 200, base=3)") mock_experiment(monkeypatch, y=[1, 3**2, 1, 3**4]) with create_experiment(config, trial_config) as (_, _, experiment): plot = partial_dependencies(experiment, n_grid_points=5, model_kwargs=dict(random_state=1)) assert_partial_dependencies_plot(plot, dims=["x", "y"], log_dims=["y"])
def test_multidim(self, monkeypatch): """Tests that dimensions with shape > 1 are flattened properly""" mock_train_regressor(monkeypatch) config = mock_space(y="uniform(0, 3, shape=2)") mock_experiment(monkeypatch, y=[[3, 3], [2, 3], [1, 2], [0, 3]]) with create_experiment(config, trial_config) as (_, _, experiment): plot = partial_dependencies(experiment, n_grid_points=5, model_kwargs=dict(random_state=1)) assert_partial_dependencies_plot(plot, dims=["x", "y[0]", "y[1]"])
def test_one_param(self, monkeypatch): """Test ploting a space with only 1 dim""" mock_train_regressor(monkeypatch) config = mock_space(y=None) mock_experiment(monkeypatch, y="drop") with create_experiment(config, trial_config) as (_, _, experiment): plot = partial_dependencies(experiment, n_grid_points=5, model_kwargs=dict(random_state=1)) assert_partial_dependencies_plot(plot, dims=["x"], n_grid_points=5)
def test_custom_model_kwargs(self, monkeypatch): """Test changing arguments of regression model""" mock_train_regressor(monkeypatch, assert_model_kwargs=dict(random_state=1)) config = mock_space() mock_experiment(monkeypatch) with create_experiment(config, trial_config) as (_, _, experiment): plot = partial_dependencies( experiment, n_grid_points=5, model_kwargs=dict(random_state=1), )
def test_returns_plotly_object(self, monkeypatch): """Tests that the plotly backend returns a plotly object""" mock_train_regressor(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): plot = partial_dependencies(experiment, n_grid_points=5, model_kwargs=dict(random_state=1)) assert type(plot) is plotly.graph_objects.Figure
def test_int_logarithmic_scales(self, monkeypatch): """Test that int log dims are turn to log scale""" mock_train_regressor(monkeypatch) config = mock_space(y="loguniform(1, 1000, discrete=True)", z="uniform(0, 1)") mock_experiment(monkeypatch, y=[1, 10, 100, 1000], z=[0, 0.1, 0.2, 0.5]) with create_experiment(config, trial_config) as (_, _, experiment): plot = partial_dependencies( experiment, n_grid_points=5, model_kwargs=dict(random_state=1) ) assert_partial_dependencies_plot( plot, dims=["x", "y", "z"], n_grid_points=5, log_dims=["y"] )
def test_categorical(self, monkeypatch): """Tests that categorical is supported""" mock_train_regressor(monkeypatch) config = mock_space(y='choices(["a", "b", "c"])') mock_experiment(monkeypatch, y=["c", "c", "a", "b"]) with create_experiment(config, trial_config) as (_, _, experiment): plot = partial_dependencies( experiment, n_grid_points=5, model_kwargs=dict(random_state=1) ) # There is only 3 categories, so test must be adjusted accordingly. assert_partial_dependencies_plot( plot, dims=["x", "y"], n_grid_points={"x": 5, "y": 3} )
def test_select_params(self, monkeypatch): """Test selecting subset""" mock_train_regressor(monkeypatch) config = mock_space(z="uniform(0, 1)") mock_experiment(monkeypatch, z=[0, 0.1, 0.2, 0.5]) for params in [["x"], ["x", "y"], ["y", "z"]]: with create_experiment(config, trial_config) as (_, _, experiment): plot = partial_dependencies( experiment, params=params, n_grid_points=5, model_kwargs=dict(random_state=1), ) assert_partial_dependencies_plot(plot, dims=params, n_grid_points=5)
def test_custom_n_grid_points(self, monkeypatch): """Test changing n_grid_points value""" mock_train_regressor(monkeypatch) config = mock_space() mock_experiment(monkeypatch) with create_experiment(config, trial_config) as (_, _, experiment): plot = partial_dependencies( experiment, n_grid_points=10, model_kwargs=dict(random_state=1), ) with pytest.raises(AssertionError): assert_partial_dependencies_plot(plot, dims=["x", "y"], n_grid_points=5) assert_partial_dependencies_plot(plot, dims=["x", "y"], n_grid_points=10)
def test_logarithmic_scales_first(self, monkeypatch): """Test that log dims are turn to log scale Test first dim specifically because special xaxis name for first dim. """ mock_train_regressor(monkeypatch) config = mock_space(x="loguniform(0.001, 1)", z="uniform(0, 1)") mock_experiment(monkeypatch, x=[0.001, 0.1, 0.01, 1], z=[0, 0.1, 0.2, 0.5]) with create_experiment(config, trial_config) as (_, _, experiment): plot = partial_dependencies( experiment, n_grid_points=5, model_kwargs=dict(random_state=1) ) assert_partial_dependencies_plot( plot, dims=["x", "y", "z"], n_grid_points=5, log_dims=["x"] )
def test_graph_layout(self, monkeypatch): """Tests the layout of the plot""" mock_train_regressor(monkeypatch) config = mock_space() mock_experiment(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): plot = partial_dependencies(experiment, n_grid_points=5, model_kwargs=dict(random_state=1)) df = experiment.to_pandas() assert df["x"].tolist() == [0, 1, 2, 4] assert df["y"].tolist() == [3, 2, 0, 1] assert df["objective"].tolist() == [0.1, 0.2, 0.3, 0.5] assert_partial_dependencies_plot(plot, dims=["x", "y"])