def test_returns_plotly_object(self, monkeypatch): """Tests that the plotly backend returns a plotly object""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): plot = regrets([experiment]) assert type(plot) is plotly.graph_objects.Figure
def test_graph_layout(self, monkeypatch): """Tests the layout of the plot""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): plot = regrets([experiment]) assert_regrets_plot(plot, [f"{experiment.name}-v{experiment.version}"])
def test_dict_of_experiments(self, monkeypatch): """Tests the regrets with renamed experiments""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): plot = regrets({"exp-1": experiment, "exp-2": experiment}) assert_regrets_plot(plot, ["exp-1", "exp-2"])
def test_unbalanced_experiments(self, monkeypatch): """Tests the regrets with avg of unbalanced experiments""" mock_experiment_with_random_to_pandas(monkeypatch, unbalanced=True) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): plot = regrets({"exp-1": [experiment] * 10, "exp-2": [experiment] * 10}) assert_regrets_plot(plot, ["exp-1", "exp-2"], with_avg=True, balanced=0)
def test_list_of_experiments(self, monkeypatch): """Tests the regrets with list of experiments""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): child = orion.client.create_experiment( experiment.name, branching={"branch_to": "child"}) plot = regrets([experiment, child]) # Exps are sorted alphabetically by names. assert_regrets_plot( plot, [f"{exp.name}-v{exp.version}" for exp in [child, experiment]])
def analysis(self, task, experiments): """ Generate a `plotly.graph_objects.Figure` to display average performance for each search algorithm. task: str Name of the task experiments: list A list of (task_index, experiment), where task_index is the index of task to run for this assessment, and experiment is an instance of `orion.core.worker.experiment`. """ algorithm_groups = defaultdict(list) for _, exp in experiments: algorithm_name = list(exp.configuration["algorithms"].keys())[0] algorithm_groups[algorithm_name].append(exp) return regrets(algorithm_groups)
def test_ignore_uncompleted_statuses(self, monkeypatch): """Tests that uncompleted statuses are ignored""" mock_experiment_with_random_to_pandas( monkeypatch, status=[ "completed", "new", "reserved", "completed", "broken", "completed", "interrupted", "completed", ], ) with create_experiment(config, trial_config) as (_, _, experiment): plot = regrets([experiment]) assert_regrets_plot(plot, [f"{experiment.name}-v{experiment.version}"], balanced=4)
def test_list_of_experiments_name_conflict(self, monkeypatch): """Tests the regrets with list of experiments with the same name""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): child = orion.client.create_experiment( experiment.name, branching={"branch_to": experiment.name, "enable": True}, ) assert child.name == experiment.name assert child.version == experiment.version + 1 plot = regrets([experiment, child]) # Exps are sorted alphabetically by names. assert_regrets_plot( plot, [f"{exp.name}-v{exp.version}" for exp in [experiment, child]] )
def analysis(self, task, experiments): """ Generate a `plotly.graph_objects.Figure` to display average performance for each search algorithm. task: str Name of the task experiments: list A list of (task_index, experiment), where task_index is the index of task to run for this assessment, and experiment is an instance of `orion.core.worker.experiment`. """ algorithm_groups = defaultdict(list) algorithm_worker_groups = defaultdict(list) for task_index, exp in experiments: algorithm_name = list(exp.configuration["algorithms"].keys())[0] algorithm_groups[algorithm_name].append(exp) n_worker = self.workers[task_index] algo_key = algorithm_name + "_workers_" + str(n_worker) algorithm_worker_groups[algo_key].append(exp) assessment = self.__class__.__name__ figure = defaultdict(dict) figure[assessment][task] = dict() figure[assessment][task][ parallel_assessment.__name__] = parallel_assessment( algorithm_groups) figure[assessment][task][durations.__name__] = durations( algorithm_worker_groups) figure[assessment][task][regrets.__name__] = regrets( algorithm_worker_groups) return figure
def test_requires_argument(self): """Tests that the experiment data are required.""" with pytest.raises(ValueError): regrets(None)
def test_unsupported_order_key(self): """Tests that unsupported order keys are rejected""" with create_experiment(config, trial_config) as (_, _, experiment): with pytest.raises(ValueError): regrets([experiment], order_by="unsupported")