def test_custom_n_samples(self, monkeypatch):
"""Test changing n_samples value"""
mock_train_regressor(monkeypatch)
config = mock_space()
mock_experiment(monkeypatch)
PARAMS = ["x", "y"]
N_SAMPLES = numpy.random.randint(20, 50)
def mock_partial_dependency_grid(space, model, params, samples,
n_points):
print(samples)
assert samples.shape == (N_SAMPLES, len(PARAMS))
return partial_dependency_grid(space, model, params, samples,
n_points)
monkeypatch.setattr(
"orion.analysis.partial_dependency_utils.partial_dependency_grid",
mock_partial_dependency_grid,
)
with create_experiment(config, trial_config) as (_, _, experiment):
plot = partial_dependencies(
experiment,
n_grid_points=10,
model_kwargs=dict(random_state=1),
n_samples=N_SAMPLES,
)
def test_categorical_multidim(self, monkeypatch):
"""Tests that multidim categorical is supported"""
mock_train_regressor(monkeypatch)
config = mock_space(y='choices(["a", "b", "c"], shape=3)')
mock_experiment(
monkeypatch,
y=[["c", "b", "a"], ["c", "a", "c"], ["a", "b", "a"],
["c", "b", "b"]],
)
with create_experiment(config, trial_config) as (_, _, experiment):
plot = partial_dependencies(experiment,
n_grid_points=5,
model_kwargs=dict(random_state=1))
assert_partial_dependencies_plot(
plot,
dims=["x", "y[0]", "y[1]", "y[2]"],
n_grid_points={
"x": 5,
"y[0]": 3,
"y[1]": 3,
"y[2]": 3
},
)
def test_ignore_uncompleted_statuses(self, monkeypatch):
"""Tests that uncompleted statuses are ignored"""
mock_train_regressor(monkeypatch)
config = mock_space()
mock_experiment(
monkeypatch,
ids="abcdefgh",
x=[0, 0, 0, 1, 0, 2, 0, 3],
y=[1, 0, 0, 2, 0, 0, 0, 3],
objectives=[0.1, None, None, 0.2, None, 0.3, None, 0.5],
status=[
"completed",
"new",
"reserved",
"completed",
"broken",
"completed",
"interrupted",
"completed",
],
)
with create_experiment(config, trial_config) as (_, _, experiment):
plot = partial_dependencies(experiment, n_grid_points=5)
assert_partial_dependencies_plot(plot, dims=["x", "y"])
def test_fidelity(self, monkeypatch):
"""Tests that fidelity is supported"""
mock_train_regressor(monkeypatch)
config = mock_space(y="fidelity(1, 200, base=3)")
mock_experiment(monkeypatch, y=[1, 3**2, 1, 3**4])
with create_experiment(config, trial_config) as (_, _, experiment):
plot = partial_dependencies(experiment,
n_grid_points=5,
model_kwargs=dict(random_state=1))
assert_partial_dependencies_plot(plot, dims=["x", "y"], log_dims=["y"])
def test_multidim(self, monkeypatch):
"""Tests that dimensions with shape > 1 are flattened properly"""
mock_train_regressor(monkeypatch)
config = mock_space(y="uniform(0, 3, shape=2)")
mock_experiment(monkeypatch, y=[[3, 3], [2, 3], [1, 2], [0, 3]])
with create_experiment(config, trial_config) as (_, _, experiment):
plot = partial_dependencies(experiment,
n_grid_points=5,
model_kwargs=dict(random_state=1))
assert_partial_dependencies_plot(plot, dims=["x", "y[0]", "y[1]"])
def test_one_param(self, monkeypatch):
"""Test ploting a space with only 1 dim"""
mock_train_regressor(monkeypatch)
config = mock_space(y=None)
mock_experiment(monkeypatch, y="drop")
with create_experiment(config, trial_config) as (_, _, experiment):
plot = partial_dependencies(experiment,
n_grid_points=5,
model_kwargs=dict(random_state=1))
assert_partial_dependencies_plot(plot, dims=["x"], n_grid_points=5)
def test_custom_model_kwargs(self, monkeypatch):
"""Test changing arguments of regression model"""
mock_train_regressor(monkeypatch, assert_model_kwargs=dict(random_state=1))
config = mock_space()
mock_experiment(monkeypatch)
with create_experiment(config, trial_config) as (_, _, experiment):
plot = partial_dependencies(
experiment,
n_grid_points=5,
model_kwargs=dict(random_state=1),
)
def test_returns_plotly_object(self, monkeypatch):
"""Tests that the plotly backend returns a plotly object"""
mock_train_regressor(monkeypatch)
with create_experiment(config, trial_config, ["completed"]) as (
_,
_,
experiment,
):
plot = partial_dependencies(experiment,
n_grid_points=5,
model_kwargs=dict(random_state=1))
assert type(plot) is plotly.graph_objects.Figure
def test_int_logarithmic_scales(self, monkeypatch):
"""Test that int log dims are turn to log scale"""
mock_train_regressor(monkeypatch)
config = mock_space(y="loguniform(1, 1000, discrete=True)", z="uniform(0, 1)")
mock_experiment(monkeypatch, y=[1, 10, 100, 1000], z=[0, 0.1, 0.2, 0.5])
with create_experiment(config, trial_config) as (_, _, experiment):
plot = partial_dependencies(
experiment, n_grid_points=5, model_kwargs=dict(random_state=1)
)
assert_partial_dependencies_plot(
plot, dims=["x", "y", "z"], n_grid_points=5, log_dims=["y"]
)
def test_categorical(self, monkeypatch):
"""Tests that categorical is supported"""
mock_train_regressor(monkeypatch)
config = mock_space(y='choices(["a", "b", "c"])')
mock_experiment(monkeypatch, y=["c", "c", "a", "b"])
with create_experiment(config, trial_config) as (_, _, experiment):
plot = partial_dependencies(
experiment, n_grid_points=5, model_kwargs=dict(random_state=1)
)
# There is only 3 categories, so test must be adjusted accordingly.
assert_partial_dependencies_plot(
plot, dims=["x", "y"], n_grid_points={"x": 5, "y": 3}
)
def test_select_params(self, monkeypatch):
"""Test selecting subset"""
mock_train_regressor(monkeypatch)
config = mock_space(z="uniform(0, 1)")
mock_experiment(monkeypatch, z=[0, 0.1, 0.2, 0.5])
for params in [["x"], ["x", "y"], ["y", "z"]]:
with create_experiment(config, trial_config) as (_, _, experiment):
plot = partial_dependencies(
experiment,
params=params,
n_grid_points=5,
model_kwargs=dict(random_state=1),
)
assert_partial_dependencies_plot(plot, dims=params, n_grid_points=5)
def test_custom_n_grid_points(self, monkeypatch):
"""Test changing n_grid_points value"""
mock_train_regressor(monkeypatch)
config = mock_space()
mock_experiment(monkeypatch)
with create_experiment(config, trial_config) as (_, _, experiment):
plot = partial_dependencies(
experiment,
n_grid_points=10,
model_kwargs=dict(random_state=1),
)
with pytest.raises(AssertionError):
assert_partial_dependencies_plot(plot, dims=["x", "y"], n_grid_points=5)
assert_partial_dependencies_plot(plot, dims=["x", "y"], n_grid_points=10)
def test_logarithmic_scales_first(self, monkeypatch):
"""Test that log dims are turn to log scale
Test first dim specifically because special xaxis name for first dim.
"""
mock_train_regressor(monkeypatch)
config = mock_space(x="loguniform(0.001, 1)", z="uniform(0, 1)")
mock_experiment(monkeypatch, x=[0.001, 0.1, 0.01, 1], z=[0, 0.1, 0.2, 0.5])
with create_experiment(config, trial_config) as (_, _, experiment):
plot = partial_dependencies(
experiment, n_grid_points=5, model_kwargs=dict(random_state=1)
)
assert_partial_dependencies_plot(
plot, dims=["x", "y", "z"], n_grid_points=5, log_dims=["x"]
)
def test_graph_layout(self, monkeypatch):
"""Tests the layout of the plot"""
mock_train_regressor(monkeypatch)
config = mock_space()
mock_experiment(monkeypatch)
with create_experiment(config, trial_config, ["completed"]) as (
_,
_,
experiment,
):
plot = partial_dependencies(experiment,
n_grid_points=5,
model_kwargs=dict(random_state=1))
df = experiment.to_pandas()
assert df["x"].tolist() == [0, 1, 2, 4]
assert df["y"].tolist() == [3, 2, 0, 1]
assert df["objective"].tolist() == [0.1, 0.2, 0.3, 0.5]
assert_partial_dependencies_plot(plot, dims=["x", "y"])
