def test_dict_of_experiments(self, monkeypatch): """Tests the regrets with renamed experiments""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): plot = regrets({"exp-1": experiment, "exp-2": experiment}) assert_regrets_plot(plot, ["exp-1", "exp-2"])
def test_graph_layout(self, monkeypatch): """Tests the layout of the plot""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): plot = regrets([experiment]) assert_regrets_plot(plot, [f"{experiment.name}-v{experiment.version}"])
def test_unbalanced_experiments(self, monkeypatch): """Tests the regrets with avg of unbalanced experiments""" mock_experiment_with_random_to_pandas(monkeypatch, unbalanced=True) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): plot = regrets({"exp-1": [experiment] * 10, "exp-2": [experiment] * 10}) assert_regrets_plot(plot, ["exp-1", "exp-2"], with_avg=True, balanced=0)
def test_figure_layout(self, study_experiments_config): """Test assessment plot format""" ar1 = AverageResult() with create_study_experiments(**study_experiments_config) as experiments: plot = ar1.analysis("task_name", experiments) assert_regrets_plot( plot, [ list(algorithm["algorithm"].keys())[0] for algorithm in study_experiments_config["algorithms"] ], balanced=study_experiments_config["max_trial"], with_avg=True, )
def test_list_of_experiments(self, monkeypatch): """Tests the regrets with list of experiments""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): child = orion.client.create_experiment( experiment.name, branching={"branch_to": "child"}) plot = regrets([experiment, child]) # Exps are sorted alphabetically by names. assert_regrets_plot( plot, [f"{exp.name}-v{exp.version}" for exp in [child, experiment]])
def test_ignore_uncompleted_statuses(self, monkeypatch): """Tests that uncompleted statuses are ignored""" mock_experiment_with_random_to_pandas( monkeypatch, status=[ "completed", "new", "reserved", "completed", "broken", "completed", "interrupted", "completed", ], ) with create_experiment(config, trial_config) as (_, _, experiment): plot = regrets([experiment]) assert_regrets_plot(plot, [f"{experiment.name}-v{experiment.version}"], balanced=4)
def test_list_of_experiments_name_conflict(self, monkeypatch): """Tests the regrets with list of experiments with the same name""" mock_experiment_with_random_to_pandas(monkeypatch) with create_experiment(config, trial_config, ["completed"]) as ( _, _, experiment, ): child = orion.client.create_experiment( experiment.name, branching={"branch_to": experiment.name, "enable": True}, ) assert child.name == experiment.name assert child.version == experiment.version + 1 plot = regrets([experiment, child]) # Exps are sorted alphabetically by names. assert_regrets_plot( plot, [f"{exp.name}-v{exp.version}" for exp in [experiment, child]] )
def test_analysis(self, study_experiments_config): """Test assessment plot format""" task_num = 2 n_workers = [1, 2, 4] pa1 = ParallelAssessment(task_num=task_num, n_workers=n_workers) study_experiments_config["task_number"] = task_num study_experiments_config["n_workers"] = n_workers with create_study_experiments( **study_experiments_config) as experiments: figure = pa1.analysis("task_name", experiments) names = [] algorithms = [] for algorithm in study_experiments_config["algorithms"]: algo = list(algorithm["algorithm"].keys())[0] algorithms.append(algo) for worker in n_workers: names.append((algo + "_workers_" + str(worker))) assert len(figure["ParallelAssessment"]["task_name"]) == 3 assert_regrets_plot( figure["ParallelAssessment"]["task_name"]["regrets"], names, balanced=study_experiments_config["max_trial"], with_avg=True, ) asset_parallel_assessment_plot( figure["ParallelAssessment"]["task_name"] ["parallel_assessment"], algorithms, 3, ) assert_durations_plot( figure["ParallelAssessment"]["task_name"]["durations"], names)