def test_best_result_no_report(ray_start_2_cpus): def f(config): pass analysis = tune.run(f, config={"x": tune.grid_search([1, 2])}) result_grid = ResultGrid(analysis) with pytest.raises(RuntimeError, match="No best trial found*"): result_grid.get_best_result(metric="x", mode="max")
def test_best_result_checkpoint_history(ray_start_2_cpus): def f(config): for i in range(2): with tune.checkpoint_dir(step=i) as checkpoint_dir: path = os.path.join(checkpoint_dir, "checkpoint") with open(path, "w") as f: f.write(json.dumps(dict(x=config["x"], step=i))) tune.report(x=config["x"], step=i) analysis = tune.run(f, config={"x": tune.grid_search([1, 3])}) # No checkpointing config. Use metric and mode result_grid = ResultGrid(analysis) best_result = result_grid.get_best_result(metric="x", mode="max") assert best_result.metrics["x"] == 3 print(best_result.best_checkpoints) print([ x[0].get_internal_representation() for x in best_result.best_checkpoints ]) assert len(best_result.best_checkpoints) == 2 i = 0 for checkpoint, metrics in best_result.best_checkpoints: assert isinstance(checkpoint, Checkpoint) assert metrics["x"] == 3 assert metrics["step"] == i i += 1
def test_best_result(ray_start_2_cpus): def f(config): for _ in range(2): tune.report(x=config["x"]) analysis = tune.run(f, config={"x": tune.grid_search([1, 2])}) result_grid = ResultGrid(analysis) best_result = result_grid.get_best_result(metric="x", mode="max") assert best_result.config["x"] == 2 assert best_result.metrics["x"] == 2
def test_no_metric_mode(ray_start_2_cpus): def f(config): tune.report(x=1) analysis = tune.run(f) result_grid = ResultGrid(analysis) with pytest.raises(ValueError): result_grid.get_best_result() with pytest.raises(ValueError): result_grid.get_best_result(metric="x") with pytest.raises(ValueError): result_grid.get_best_result(mode="max")