def test_status( self, study, study_experiments_config, task_number, max_trial, ): """Test to get status of a study""" with create_study_experiments(**study_experiments_config) as experiments: study.experiments_info = experiments assert study.status() == [ { "algorithm": "random", "assessment": "AverageResult", "task": "RosenBrock", "experiments": task_number, "completed": task_number, "trials": task_number * max_trial, }, { "algorithm": "tpe", "assessment": "AverageResult", "task": "RosenBrock", "experiments": task_number, "completed": task_number, "trials": task_number * max_trial, }, ]
def test_status( self, benchmark, study, study_experiments_config, task_number, max_trial, ): """Test to get the status of a benchmark""" with create_study_experiments(**study_experiments_config) as experiments: study.experiments_info = experiments benchmark.studies = [study] assert benchmark.status() == [ { "Algorithms": "random", "Assessments": "AverageResult", "Tasks": "RosenBrock", "Total Experiments": task_number, "Completed Experiments": task_number, "Submitted Trials": task_number * max_trial, }, { "Algorithms": "tpe", "Assessments": "AverageResult", "Tasks": "RosenBrock", "Total Experiments": task_number, "Completed Experiments": task_number, "Submitted Trials": task_number * max_trial, }, ]
def test_analysis(self, benchmark, study, study_experiments_config): """Test to analysis benchmark result""" with create_study_experiments(**study_experiments_config) as experiments: study.experiments_info = experiments benchmark.studies = [study] figures = benchmark.analysis() assert len(figures) == 1 assert type(figures[0]) is plotly.graph_objects.Figure
def test_experiments(self, study, study_experiments_config, task_number): """Test to get experiments of a study""" algo_num = len(study_experiments_config["algorithms"]) with create_study_experiments(**study_experiments_config) as experiments: study.experiments_info = experiments experiments = study.experiments() assert ( len(experiments) == study_experiments_config["task_number"] * algo_num ) assert isinstance(experiments[0], Experiment)
def test_analysis( self, study, study_experiments_config, ): """Test to get the ploty figure of a study""" with create_study_experiments(**study_experiments_config) as experiments: study.experiments_info = experiments plot = study.analysis() assert type(plot) is plotly.graph_objects.Figure
def test_figure_layout(self, study_experiments_config): """Test assessment plot format""" ar1 = AverageResult() with create_study_experiments(**study_experiments_config) as experiments: plot = ar1.analysis("task_name", experiments) assert_regrets_plot( plot, [ list(algorithm["algorithm"].keys())[0] for algorithm in study_experiments_config["algorithms"] ], balanced=study_experiments_config["max_trial"], with_avg=True, )
def test_analysis( self, study, study_experiments_config, ): """Test to get the ploty figure of a study""" with create_study_experiments(**study_experiments_config) as experiments: study.experiments_info = experiments figure = study.analysis() assert ( type(figure[study.assess_name][study.task_name]["regrets"]) is plotly.graph_objects.Figure )
def test_experiments( self, benchmark, study, study_experiments_config, max_trial, ): """Test to get experiments list of a benchmark""" with create_study_experiments( **study_experiments_config) as experiments: study.experiments_info = experiments benchmark.studies = [study] assert benchmark.experiments() == [ { "Algorithm": "random", "Experiment Name": "experiment-name-0", "Number Trial": max_trial, "Best Evaluation": 0, }, { "Algorithm": "tpe", "Experiment Name": "experiment-name-1", "Number Trial": max_trial, "Best Evaluation": 0, }, { "Algorithm": "random", "Experiment Name": "experiment-name-2", "Number Trial": max_trial, "Best Evaluation": 0, }, { "Algorithm": "tpe", "Experiment Name": "experiment-name-3", "Number Trial": max_trial, "Best Evaluation": 0, }, ]
def test_analysis(self, study_experiments_config): """Test assessment plot format""" task_num = 2 n_workers = [1, 2, 4] pa1 = ParallelAssessment(task_num=task_num, n_workers=n_workers) study_experiments_config["task_number"] = task_num study_experiments_config["n_workers"] = n_workers with create_study_experiments( **study_experiments_config) as experiments: figure = pa1.analysis("task_name", experiments) names = [] algorithms = [] for algorithm in study_experiments_config["algorithms"]: algo = list(algorithm["algorithm"].keys())[0] algorithms.append(algo) for worker in n_workers: names.append((algo + "_workers_" + str(worker))) assert len(figure["ParallelAssessment"]["task_name"]) == 3 assert_regrets_plot( figure["ParallelAssessment"]["task_name"]["regrets"], names, balanced=study_experiments_config["max_trial"], with_avg=True, ) asset_parallel_assessment_plot( figure["ParallelAssessment"]["task_name"] ["parallel_assessment"], algorithms, 3, ) assert_durations_plot( figure["ParallelAssessment"]["task_name"]["durations"], names)