def test_dict_of_experiments(self, monkeypatch):
"""Tests the rankings with renamed experiments"""
mock_experiment_with_random_to_pandas(monkeypatch)
with create_experiment(config, trial_config, ["completed"]) as (
_,
_,
experiment,
):
plot = rankings({"exp-1": experiment, "exp-2": experiment})
assert_rankings_plot(plot, ["exp-1", "exp-2"])
def test_unbalanced_experiments(self, monkeypatch):
"""Tests the regrets with avg of unbalanced experiments"""
mock_experiment_with_random_to_pandas(monkeypatch, unbalanced=True)
with create_experiment(config, trial_config, ["completed"]) as (
_,
_,
experiment,
):
plot = rankings({"exp-1": [experiment] * 10, "exp-2": [experiment] * 10})
assert_rankings_plot(plot, ["exp-1", "exp-2"], with_avg=True, balanced=0)
def test_returns_plotly_object(self, monkeypatch):
"""Tests that the plotly backend returns a plotly object"""
mock_experiment_with_random_to_pandas(monkeypatch)
with create_experiment(config, trial_config, ["completed"]) as (
_,
_,
experiment,
):
plot = rankings([experiment, experiment])
assert type(plot) is plotly.graph_objects.Figure
def test_dict_of_list_of_experiments(self, monkeypatch):
"""Tests the rankings with avg of experiments separated in lists"""
mock_experiment_with_random_to_pandas(monkeypatch)
with create_experiment(config, trial_config, ["completed"]) as (
_,
_,
experiment,
):
plot = rankings({"exp-1": [experiment] * 10, "exp-2": [experiment] * 10})
assert_rankings_plot(plot, ["exp-1", "exp-2"], with_avg=True)
def test_graph_layout(self, monkeypatch):
"""Tests the layout of the plot"""
mock_experiment_with_random_to_pandas(monkeypatch)
with create_experiment(config, trial_config, ["completed"]) as (
_,
_,
experiment,
):
plot = rankings([experiment])
assert_rankings_plot(plot, [f"{experiment.name}-v{experiment.version}"])
def test_list_of_dict_of_experiments(self, monkeypatch):
"""Tests the rankings with avg of competitions"""
mock_experiment_with_random_to_pandas(monkeypatch)
with create_experiment(config, trial_config, ["completed"]) as (
_,
_,
experiment,
):
plot = rankings(
[{"exp-1": experiment, "exp-2": experiment} for _ in range(10)]
)
assert_rankings_plot(plot, ["exp-1", "exp-2"], with_avg=True)
def test_list_of_experiments(self, monkeypatch):
"""Tests the rankings with list of experiments"""
mock_experiment_with_random_to_pandas(monkeypatch)
with create_experiment(config, trial_config, ["completed"]) as (
_,
_,
experiment,
):
child = orion.client.create_experiment(
experiment.name, branching={"branch_to": "child"})
plot = rankings([experiment, child])
# Exps are sorted alphabetically by names.
assert_rankings_plot(
plot,
[f"{exp.name}-v{exp.version}" for exp in [child, experiment]])
def analysis(self, task, experiments):
"""
Generate a `plotly.graph_objects.Figure` to display average rankings
between different search algorithms.
task: str
Name of the task
experiments: list
A list of (task_index, experiment), where task_index is the index of task to run for
this assessment, and experiment is an instance of `orion.core.worker.experiment`.
"""
algorithm_groups = defaultdict(list)
for _, exp in experiments:
algorithm_name = list(exp.configuration["algorithms"].keys())[0]
algorithm_groups[algorithm_name].append(exp)
return rankings(algorithm_groups)
def test_list_of_experiments_name_conflict(self, monkeypatch):
"""Tests the rankings with list of experiments with the same name"""
mock_experiment_with_random_to_pandas(monkeypatch)
with create_experiment(config, trial_config, ["completed"]) as (
_,
_,
experiment,
):
child = orion.client.create_experiment(
experiment.name,
branching={"branch_to": experiment.name, "enable": True},
)
assert child.name == experiment.name
assert child.version == experiment.version + 1
plot = rankings([experiment, child])
# Exps are sorted alphabetically by names.
assert_rankings_plot(
plot, [f"{exp.name}-v{exp.version}" for exp in [experiment, child]]
)
def test_ignore_uncompleted_statuses(self, monkeypatch):
"""Tests that uncompleted statuses are ignored"""
mock_experiment_with_random_to_pandas(
monkeypatch,
status=[
"completed",
"new",
"reserved",
"completed",
"broken",
"completed",
"interrupted",
"completed",
],
)
with create_experiment(config, trial_config) as (_, _, experiment):
plot = rankings([experiment])
assert_rankings_plot(plot,
[f"{experiment.name}-v{experiment.version}"],
balanced=4)
def test_unsupported_order_key(self):
"""Tests that unsupported order keys are rejected"""
with create_experiment(config, trial_config) as (_, _, experiment):
with pytest.raises(ValueError):
rankings([experiment], order_by="unsupported")
def test_requires_argument(self):
"""Tests that the experiment data are required."""
with pytest.raises(ValueError):
rankings(None)
