def test_dict_of_experiments(self, monkeypatch): """Tests the rankings with renamed experiments""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): plot = rankings({"exp-1": experiment, "exp-2": experiment}) assert_rankings_plot(plot, ["exp-1", "exp-2"])
def test_unbalanced_experiments(self, monkeypatch): """Tests the regrets with avg of unbalanced experiments""" mock_experiment_with_random_to_pandas(monkeypatch, unbalanced=True) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): plot = rankings({"exp-1": [experiment] * 10, "exp-2": [experiment] * 10}) assert_rankings_plot(plot, ["exp-1", "exp-2"], with_avg=True, balanced=0)
def test_returns_plotly_object(self, monkeypatch): """Tests that the plotly backend returns a plotly object""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): plot = rankings([experiment, experiment]) assert type(plot) is plotly.graph_objects.Figure
def test_dict_of_list_of_experiments(self, monkeypatch): """Tests the rankings with avg of experiments separated in lists""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): plot = rankings({"exp-1": [experiment] * 10, "exp-2": [experiment] * 10}) assert_rankings_plot(plot, ["exp-1", "exp-2"], with_avg=True)
def test_graph_layout(self, monkeypatch): """Tests the layout of the plot""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): plot = rankings([experiment]) assert_rankings_plot(plot, [f"{experiment.name}-v{experiment.version}"])
def test_list_of_dict_of_experiments(self, monkeypatch): """Tests the rankings with avg of competitions""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): plot = rankings( [{"exp-1": experiment, "exp-2": experiment} for _ in range(10)] ) assert_rankings_plot(plot, ["exp-1", "exp-2"], with_avg=True)
def test_list_of_experiments(self, monkeypatch): """Tests the rankings with list of experiments""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): child = orion.client.create_experiment( experiment.name, branching={"branch_to": "child"}) plot = rankings([experiment, child]) # Exps are sorted alphabetically by names. assert_rankings_plot( plot, [f"{exp.name}-v{exp.version}" for exp in [child, experiment]])
def analysis(self, task, experiments): """ Generate a `plotly.graph_objects.Figure` to display average rankings between different search algorithms. task: str Name of the task experiments: list A list of (task_index, experiment), where task_index is the index of task to run for this assessment, and experiment is an instance of `orion.core.worker.experiment`. """ algorithm_groups = defaultdict(list) for _, exp in experiments: algorithm_name = list(exp.configuration["algorithms"].keys())[0] algorithm_groups[algorithm_name].append(exp) return rankings(algorithm_groups)
def test_list_of_experiments_name_conflict(self, monkeypatch): """Tests the rankings with list of experiments with the same name""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): child = orion.client.create_experiment( experiment.name, branching={"branch_to": experiment.name, "enable": True}, ) assert child.name == experiment.name assert child.version == experiment.version + 1 plot = rankings([experiment, child]) # Exps are sorted alphabetically by names. assert_rankings_plot( plot, [f"{exp.name}-v{exp.version}" for exp in [experiment, child]] )
def test_ignore_uncompleted_statuses(self, monkeypatch): """Tests that uncompleted statuses are ignored""" mock_experiment_with_random_to_pandas( monkeypatch, status=[ "completed", "new", "reserved", "completed", "broken", "completed", "interrupted", "completed", ], ) with create_experiment(config, trial_config) as (_, _, experiment): plot = rankings([experiment]) assert_rankings_plot(plot, [f"{experiment.name}-v{experiment.version}"], balanced=4)
def test_unsupported_order_key(self): """Tests that unsupported order keys are rejected""" with create_experiment(config, trial_config) as (_, _, experiment): with pytest.raises(ValueError): rankings([experiment], order_by="unsupported")
def test_requires_argument(self): """Tests that the experiment data are required.""" with pytest.raises(ValueError): rankings(None)