def test_create_experiment_debug_mode(self, tmp_path, benchmark_config_py): """Test that EphemeralDB is used in debug mode whatever the storage config given""" update_singletons() conf_file = str(tmp_path / "db.pkl") config = copy.deepcopy(benchmark_config_py) config["storage"] = { "type": "legacy", "database": {"type": "pickleddb", "host": conf_file}, } get_or_create_benchmark(**config) storage = get_storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, PickledDB) update_singletons() config["storage"] = {"type": "legacy", "database": {"type": "pickleddb"}} config["debug"] = True get_or_create_benchmark(**config) storage = get_storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, EphemeralDB)
def test_create_with_different_configure(self, benchmark_config_py, caplog): """Test creation with same name but different configure""" with OrionState(): config = copy.deepcopy(benchmark_config_py) bm1 = get_or_create_benchmark(**config) config = copy.deepcopy(benchmark_config_py) config["targets"][0]["assess"] = [AverageResult(2)] with caplog.at_level(logging.WARNING, logger="orion.benchmark.benchmark_client"): bm2 = get_or_create_benchmark(**config) assert bm2.configuration == bm1.configuration assert ( "Benchmark with same name is found but has different configuration, " "which will be used for this creation." in caplog.text) caplog.clear() config = copy.deepcopy(benchmark_config_py) config["targets"][0]["task"] = [ RosenBrock(26, dim=3), CarromTable(20) ] with caplog.at_level(logging.WARNING, logger="orion.benchmark.benchmark_client"): bm3 = get_or_create_benchmark(**config) assert bm3.configuration == bm1.configuration assert ( "Benchmark with same name is found but has different configuration, " "which will be used for this creation." in caplog.text)
def test_create_with_invalid_targets(self, benchmark_config_py): """Test creation with invalid Task and Assessment""" with OrionState(): with pytest.raises(AttributeError) as exc: config = copy.deepcopy(benchmark_config_py) config["targets"] = [{ "assess": [AverageResult(2)], "task": [DummyTask] }] get_or_create_benchmark(**config) assert "type object '{}' has no attribute ".format( "DummyTask") in str(exc.value) with pytest.raises(AttributeError) as exc: config = copy.deepcopy(benchmark_config_py) config["targets"] = [{ "assess": [DummyAssess], "task": [RosenBrock(25, dim=3)] }] get_or_create_benchmark(**config) assert "type object '{}' has no attribute ".format( "DummyAssess") in str(exc.value)
def test_create_with_invalid_algorithms(self, benchmark_config_py): """Test creation with a not existed algorithm""" with OrionState(): with pytest.raises(NotImplementedError) as exc: benchmark_config_py["algorithms"] = [{"fake_algorithm": {"seed": 1}}] get_or_create_benchmark(**benchmark_config_py) assert "Could not find implementation of BaseAlgorithm" in str(exc.value)
def test_create_with_only_name(self): """Test creation with a non-existing benchmark name""" with OrionState(): name = "bm00001" with pytest.raises(NoConfigurationError) as exc: get_or_create_benchmark(name) assert "Benchmark {} does not exist in DB".format(name) in str(exc.value)
def test_create_benchmark(self, benchmark_config, benchmark_config_py): """Test creation with valid configuration""" with OrionState(): bm1 = get_or_create_benchmark(**benchmark_config_py) bm2 = get_or_create_benchmark("bm00001") assert bm1.configuration == benchmark_config assert bm1.configuration == bm2.configuration
def test_create_with_not_exist_targets_parameters(self, benchmark_config): """Test creation with not existing assessment parameters""" benchmark_config["targets"][0]["assess"]["AverageResult"] = { "task_num": 2, "idontexist": 100, } with OrionState(benchmarks=benchmark_config): with pytest.raises(TypeError) as exc: get_or_create_benchmark(benchmark_config["name"]) assert "__init__() got an unexpected keyword argument 'idontexist'" in str( exc.value)
def test_create_race_condition(self, benchmark_config, benchmark_config_py, monkeypatch, caplog): """Test creation in race condition""" with OrionState(benchmarks=benchmark_config): def insert_race_condition(*args, **kwargs): if insert_race_condition.count == 0: data = {} else: data = benchmark_config insert_race_condition.count += 1 return data insert_race_condition.count = 0 monkeypatch.setattr(benchmark_client, "_fetch_benchmark", insert_race_condition) with caplog.at_level(logging.INFO, logger="orion.benchmark.benchmark_client"): bm = benchmark_client.get_or_create_benchmark( **benchmark_config_py) assert ( "Benchmark registration failed. This is likely due to a race condition. " "Now rolling back and re-attempting building it." in caplog.text) assert insert_race_condition.count == 2 del benchmark_config["_id"] assert bm.configuration == benchmark_config assert count_benchmarks() == 1
def test_create_with_executor(self, benchmark_config, benchmark_config_py): with OrionState(): config = copy.deepcopy(benchmark_config_py) bm1 = get_or_create_benchmark(**config) bm1.close() assert bm1.configuration == benchmark_config assert bm1.executor.n_workers == orion.core.config.worker.n_workers with Joblib(n_workers=2, backend="threading") as executor: config["executor"] = executor bm2 = get_or_create_benchmark(**config) assert bm2.configuration == benchmark_config assert bm2.executor.n_workers == executor.n_workers assert orion.core.config.worker.n_workers != 2
def test_create_with_deterministic_algorithm(self, benchmark_config_py): algorithms = [ { "algorithm": { "random": { "seed": 1 } } }, { "algorithm": { "gridsearch": { "n_values": 50 } }, "deterministic": True }, ] with OrionState(): config = copy.deepcopy(benchmark_config_py) config["algorithms"] = algorithms bm = get_or_create_benchmark(**config) for study in bm.studies: for status in study.status(): algo = status["algorithm"] if algo == "gridsearch": assert status["experiments"] == 1 else: assert status[ "experiments"] == study.assessment.task_num
def test_create_benchmark_bad_storage(self, benchmark_config_py): """Test error message if storage is not configured properly""" name = "oopsie_bad_storage" # Make sure there is no existing storage singleton update_singletons() with pytest.raises(NotImplementedError) as exc: benchmark_config_py["storage"] = { "type": "legacy", "database": { "type": "idontexist" }, } get_or_create_benchmark(**benchmark_config_py).close() assert "Could not find implementation of Database, type = 'idontexist'" in str( exc.value)
def test_create_with_not_loaded_targets(self, benchmark_config): """Test creation with assessment or task does not exist or not loaded""" cfg_invalid_assess = copy.deepcopy(benchmark_config) cfg_invalid_assess["targets"][0]["assess"]["idontexist"] = {"task_num": 2} with OrionState(benchmarks=cfg_invalid_assess): with pytest.raises(NotImplementedError) as exc: get_or_create_benchmark(benchmark_config["name"]) assert "Could not find implementation of BaseAssess" in str(exc.value) cfg_invalid_task = copy.deepcopy(benchmark_config) cfg_invalid_task["targets"][0]["task"]["idontexist"] = {"max_trials": 2} with OrionState(benchmarks=cfg_invalid_task): with pytest.raises(NotImplementedError) as exc: get_or_create_benchmark(benchmark_config["name"]) assert "Could not find implementation of BaseTask" in str(exc.value)
def test_create_with_invalid_algorithms(self, benchmark_config_py): """Test creation with a not existed algorithm""" with OrionState(): with pytest.raises(NotImplementedError) as exc: benchmark_config_py["algorithms"] = [{ "algorithm": { "fake_algorithm": { "seed": 1 } } }] # Pass executor to close it properly with Joblib(n_workers=2, backend="threading") as executor: get_or_create_benchmark(**benchmark_config_py, executor=executor) assert "Could not find implementation of BaseAlgorithm" in str( exc.value)
def test_experiments_parallel(self, benchmark_config_py, monkeypatch): import multiprocessing class FakeFuture: def __init__(self, value): self.value = value def wait(self, timeout=None): return def ready(self): return True def get(self, timeout=None): return self.value def successful(self): return True count = multiprocessing.Value("i", 0) is_done_value = multiprocessing.Value("i", 0) def is_done(self): return count.value > 0 def submit(*args, c=count, **kwargs): # because worker == 2 only 2 jobs were submitted # we now set is_done to True so when runner checks # for adding more jobs it will stop right away c.value += 1 return FakeFuture([dict(name="v", type="objective", value=1)]) with OrionState(): config = copy.deepcopy(benchmark_config_py) with Joblib(n_workers=5, backend="threading") as executor: monkeypatch.setattr(ExperimentClient, "is_done", property(is_done)) monkeypatch.setattr(executor, "submit", submit) config["executor"] = executor bm1 = get_or_create_benchmark(**config) client = bm1.studies[0].experiments_info[0][1] count.value = 0 bm1.process(n_workers=2) assert count.value == 2 assert executor.n_workers == 5 assert orion.core.config.worker.n_workers != 2 is_done.done = False count.value = 0 bm1.process(n_workers=3) assert count.value == 3 assert executor.n_workers == 5 assert orion.core.config.worker.n_workers != 3
def test_simple(): """Test a end 2 end exucution of benchmark""" task_num = 2 trial_num = 20 assessments = [AverageResult(task_num), AverageRank(task_num)] tasks = [ RosenBrock(trial_num, dim=3), EggHolder(trial_num, dim=4), CarromTable(trial_num), Branin(trial_num), BirdLike(trial_num), ] benchmark = get_or_create_benchmark( name="bm001", algorithms=algorithms, targets=[{ "assess": assessments, "task": tasks }], ) benchmark.process() assert len(benchmark.studies) == len(assessments) * len(tasks) status = benchmark.status() experiments = benchmark.experiments() assert len(experiments ) == len(algorithms) * task_num * len(assessments) * len(tasks) assert len(status) == len(algorithms) * len(assessments) * len(tasks) figures = benchmark.analysis() assert len(figures) == len(benchmark.studies) assert type(figures[0]) is plotly.graph_objects.Figure benchmark = get_or_create_benchmark(name="bm001") figures = benchmark.analysis() assert len(figures) == len(benchmark.studies) assert type(figures[0]) is plotly.graph_objects.Figure
def test_create_benchmark_with_storage(self, benchmark_config_py): """Test benchmark instance has the storage configurations""" config = copy.deepcopy(benchmark_config_py) storage = {"type": "legacy", "database": {"type": "EphemeralDB"}} with OrionState(storage=storage): config["storage"] = storage bm = get_or_create_benchmark(**config) bm.close() assert bm.storage_config == config["storage"]
def test_simple(): """Test a end 2 end exucution of benchmark""" task_num = 2 max_trials = 10 assessments = [AverageResult(task_num), AverageRank(task_num)] tasks = [ Branin(max_trials), BirdLike(max_trials), ] benchmark = get_or_create_benchmark( name="bm001", algorithms=algorithms, targets=[{ "assess": assessments, "task": tasks }], ) benchmark.process() assert len(benchmark.studies) == len(assessments) * len(tasks) status = benchmark.status() experiments = benchmark.experiments() assert len(experiments ) == len(algorithms) * task_num * len(assessments) * len(tasks) assert len(status) == len(algorithms) * len(assessments) * len(tasks) figures = benchmark.analysis() assert_benchmark_figures(figures, 4, assessments, tasks) benchmark = get_or_create_benchmark(name="bm001") figures = benchmark.analysis() assert_benchmark_figures(figures, 4, assessments, tasks) benchmark.close()
def test_create_benchmark_no_storage(self, benchmark_config_py): """Test creation if storage is not configured""" name = "oopsie_forgot_a_storage" host = orion.core.config.storage.database.host with OrionState(storage=orion.core.config.storage.to_dict()) as cfg: # Reset the Storage and drop instances so that get_storage() would fail. cfg.cleanup() cfg.singletons = update_singletons() # Make sure storage must be instantiated during `get_or_create_benchmark()` with pytest.raises(SingletonNotInstantiatedError): get_storage() get_or_create_benchmark(**benchmark_config_py) storage = get_storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, PickledDB) assert storage._db.host == host
def main(config: ProfetExperimentConfig): if issubclass(config.task_type, ProfetTask): task = config.task_type( max_trials=config.max_trials, task_id=0, model_config=config.profet_train_config, input_dir=config.input_dir, checkpoint_dir=config.checkpoint_dir, seed=config.seed, ) else: # NOTE: This doesn't normally happen when using this from the command-line. task = config.task_type(max_trials=config.max_trials) print(f"Storage file used: {config.storage_pickle_path}") benchmark = get_or_create_benchmark( name=config.name, algorithms=config.algorithms, targets=[{"assess": [AverageResult(config.n_repetitions)], "task": [task,],}], storage={ "type": "legacy", "database": {"type": "pickleddb", "host": str(config.storage_pickle_path)}, }, debug=config.debug, ) benchmark.setup_studies() # Since we're using the QuadraticsTask, we can evaluate the 'similarity' between # them, hence we just show these figures rather than the (very large) number of other # potential figures. figures_dir = config.figures_dir / benchmark.name figures_dir.mkdir(exist_ok=True, parents=True) benchmark.process(n_workers=1) status = benchmark.status(False) print(status) # if all(isinstance(source_task, QuadraticsTask) for source_task in benchmark.source_tasks): figures = benchmark.analysis() # # # TODO: Instead of having like 30 figures, should try and create an interactive-ish # # # plotly thingy that can switch between the different quantities. # import plotly.io as pio # pio.renderers.default = "browser" for i, figure in enumerate(figures): figure.show() figure.write_image(str(figures_dir / f"fig_{i:02}.svg")) figure.write_image(str(figures_dir / f"fig_{i:02}.png")) figure.write_html(str(figures_dir / f"fig_{i:02}.html"), include_plotlyjs="cdn")
def test_create_from_db_config(self, benchmark_config): """Test creation from existing db configubenchmark_configre""" with OrionState(benchmarks=copy.deepcopy(benchmark_config)): bm = get_or_create_benchmark(benchmark_config["name"]) assert bm.configuration == benchmark_config