def test_trial_report_heap(): t1 = Trial("a", {}) r1 = TrialReport(t1, 0.1) t2 = Trial("b", {}) r2 = TrialReport(t2, 0.2) t3 = Trial("c", {}) r3 = TrialReport(t3, 0.3) r4 = TrialReport(t3, -0.3) h = TrialReportHeap(min_heap=True) for r in [r1, r2, r3, r4]: h.push(r) assert "a" in h assert "x" not in h for r in [r4, r1, r2]: assert h.pop() is r assert 0 == len(h) h = TrialReportHeap(min_heap=False) for r in [r1, r2, r3, r4]: h.push(r) for r in [r1, r2, r4]: assert r in list(h.values()) for r in [r2, r1, r4]: assert h.pop() is r assert 0 == len(h) # test __lt__, the sort key is sort_metric! r5 = TrialReport(t1, metric=0.1, sort_metric=-0.1) r6 = TrialReport(t2, metric=0.2, sort_metric=-0.2) r7 = TrialReport(t3, metric=0.3, sort_metric=-0.3) h = TrialReportHeap(min_heap=True) for r in [r7, r6, r5]: h.push(r) for r in [r7, r6, r5]: assert h.pop() is r assert 0 == len(h) r5 = TrialReport(t1, metric=0.1, cost=0.2, rung=5) r6 = TrialReport(t2, metric=0.1, cost=0.3, rung=5) r7 = TrialReport(t3, metric=0.1, cost=0.3, rung=6) h = TrialReportHeap(min_heap=True) for r in [r7, r6, r5]: h.push(r) for r in [r5, r6, r7]: assert h.pop() is r assert 0 == len(h) # equal case r8 = TrialReport(t1, metric=0.1, cost=0.3, rung=6) r9 = TrialReport(t2, metric=0.1, cost=0.3, rung=6) h = TrialReportHeap(min_heap=False) for r in [r8, r9]: h.push(r) for r in [r8, r9]: assert h.pop() is r assert 0 == len(h)
def test_copy(): trial = Trial("abc", {"a": 1}) t1 = trial.with_dfs({"c": pd.DataFrame([[0]])}) t2 = copy.copy(t1) t3 = copy.deepcopy(t1) assert trial.trial_id == t2.trial_id == t3.trial_id assert t1.dfs is t2.dfs is t3.dfs assert 0 == len(trial.dfs) assert 1 == len(t1.dfs) assert [] == t3.keys
def test_trial(): trial = Trial("abc", {"a": 1}, {"b": 2}, keys=["a", "b"]) assert "abc" == trial.trial_id assert {"a": 1} == trial.params assert {"b": 2} == trial.metadata assert ["a", "b"] == trial.keys t2 = trial.with_params({"c": 3}) assert "abc" == t2.trial_id assert {"c": 3} == t2.params assert {"b": 2} == t2.metadata assert ["a", "b"] == t2.keys
def test_noniterative_objective(): @noniterative_objective def f1(a, b: int) -> float: return a - b assert isinstance(f1, NonIterativeObjectiveFunc) trial = Trial("abc", dict(b=20, a=10), dict(c=3)) report = f1.safe_run(trial) assert report.trial is trial assert report.metric == -10 assert report.params == trial.params assert report.metadata == {} func = to_noniterative_objective("f1") report = func.safe_run(trial) assert report.trial is trial assert report.metric == -10 assert report.params == trial.params assert report.metadata == {} @noniterative_objective(min_better=False) def f2(a, b: int) -> float: return a - b assert isinstance(f2, NonIterativeObjectiveFunc) assert not f2.min_better
def test_trial_serialization(): p = {"a": 1, "b": Rand(1, 2)} trial = Trial("abc", p, {}, keys=["x", "y"], dfs={"v": ""}) t = cloudpickle.loads(cloudpickle.dumps(trial)) assert isinstance(t.params, TuningParametersTemplate) assert ["x", "y"] == t.keys assert t.trial_id == trial.trial_id assert {"v": ""} == t.dfs
def test_report_logger(): class Mock(TrialReportLogger): def __init__(self, new_best_only: bool = False): super().__init__(new_best_only=new_best_only) self._values = [] def log(self, report: TrialReport) -> None: return self._values.append(report) @property def records(self): return self._values t1 = Trial("a", dict(a=1, b=2), keys=["x", "y"]) r1 = TrialReport(t1, 0.8, sort_metric=-0.8) t2 = Trial("b", dict(a=11, b=12), keys=["xx", "y"]) r2 = TrialReport(t2, 0.7, sort_metric=-0.7) t3 = Trial("c", dict(a=10, b=20), keys=["x", "y"]) r3 = TrialReport(t3, 0.9, sort_metric=-0.9) b = Mock() assert 0 == len(b.records) assert b.best is None assert b.on_report(r1) assert b.on_report(r3) assert r3 is b.best assert 2 == len(b.records) b = Mock() assert b.on_report(r3) assert not b.on_report(r1) assert r3 is b.best assert 2 == len(b.records) b = Mock(new_best_only=True) assert b.on_report(r3) assert not b.on_report(r1) assert r3 is b.best assert 1 == len(b.records)
def test_validator(): m = M() for cont in [True, False]: validate_iterative_objective( F(), Trial("abc", {"a": 1}), [3, 3, 2], lambda reports: [-3.0, -6.0, -8.0] == [x.sort_metric for x in reports], continuous=cont, monitor=m, ) assert 6 == len(m._reports)
def _to_trail_row(data: Dict[str, Any], metadata: Dict[str, Any]) -> Dict[str, Any]: key_names = sorted(k for k in data.keys() if not k.startswith(TUNE_PREFIX)) keys = [data[k] for k in key_names] trials: Dict[str, Trial] = {} for params in pickle.loads(data[TUNE_DATASET_PARAMS_PREFIX]): tid = to_uuid(keys, params) trials[tid] = Trial(trial_id=tid, params=params, metadata=metadata, keys=keys) data[TUNE_DATASET_TRIALS] = to_base64(list(trials.values())) del data[TUNE_DATASET_PARAMS_PREFIX] return data
def test_objective_func(tmpdir): fs = FileSystem().opendir(str(tmpdir)) j = J([3, 3, 2]) f = F().copy() t = Trial("abc", {"a": 1}) f.run(t, judge=j, checkpoint_basedir_fs=fs) assert -10 == f.v f.run(t, judge=j, checkpoint_basedir_fs=fs) assert -10 == f.v assert 6.0 == j.report.metric assert -6.0 == j.report.sort_metric f.run(t, judge=j, checkpoint_basedir_fs=fs) assert -10 == f.v assert 8.0 == j.report.metric assert -8.0 == j.report.sort_metric
def test_trial_decision(): trial = Trial("abc", {"a": 1}, {"b": Rand(0, 2)}) report = TrialReport(trial, metric=np.float(0.1), params={"c": Rand(0, 3)}, metadata={"d": 4}) decision = TrialDecision(report, budget=0.0, should_checkpoint=True, metadata={"x": 1}, reason="p") assert trial is decision.trial assert report is decision.report decision = cloudpickle.loads(cloudpickle.dumps(decision)) assert decision.should_stop assert decision.should_checkpoint assert {"x": 1} == decision.metadata assert "p" == decision.reason assert 0.0 == decision.budget assert copy.copy(decision) is decision assert copy.deepcopy(decision) is decision d2 = cloudpickle.loads(cloudpickle.dumps(decision)) assert d2.trial_id == trial.trial_id # assert Rand(0, 3) == d2.report.params["c"] assert decision.should_stop assert decision.should_checkpoint assert {"x": 1} == decision.metadata assert "p" == decision.reason decision = TrialDecision(report, budget=1.0, should_checkpoint=True, metadata={"x": 1}) assert 1.0 == decision.budget assert not decision.should_stop print(decision)
def test_to_noniterative_objective(): def f1(a, b: int) -> float: return a - b func = to_noniterative_objective(f1) assert func.min_better trial = Trial("abc", dict(b=20, a=10), dict(c=3)) report = func.safe_run(trial) assert report.trial is trial assert report.metric == -10 assert report.sort_metric == -10 assert report.params == trial.params assert report.metadata == {} func = to_noniterative_objective("f1", min_better=False) assert not func.min_better trial = Trial("abc", dict(b=20, a=10), dict(c=3)) report = func.safe_run(trial) assert report.trial is trial assert report.metric == -10 assert report.sort_metric == 10 assert report.params == trial.params assert report.metadata == {} assert -1 == func(1, 2) def f2(a, b: int) -> Tuple[float, Dict[str, Any]]: return a - b, dict(c=5) func = to_noniterative_objective(f2) trial = Trial("abc", dict(b=20, a=10), dict(c=3)) report = func.safe_run(trial) assert report.trial is trial assert report.metric == -10 assert report.params == trial.params assert report.metadata == dict(c=5) def f3(t: Trial) -> TrialReport: return TrialReport( t, t.params.simple_value["a"] - t.params.simple_value["b"], params=dict(a=1), metadata=dict(c=6), ) func = to_noniterative_objective(f3) trial = Trial("abc", dict(b=20, a=10), dict(c=3)) report = func.safe_run(trial) assert report.trial is trial assert report.metric == -10 assert report.params == dict(a=1) assert report.metadata == dict(c=6) class F4(NonIterativeObjectiveFunc): def run(self, t: Trial) -> TrialReport: return TrialReport(t, t.params["a"] - t.params["b"], params=dict(a=1), metadata=dict(c=6)) f4 = F4() f4_ = to_noniterative_objective(f4) assert isinstance(f4_, F4) assert f4 is not f4_ raises(TuneCompileError, lambda: to_noniterative_objective("abc"))
def test_trial_report(): trial = Trial("abc", {"a": Rand(3, 4)}, {"b": 2}) report = copy.copy( TrialReport( trial, metric=np.float(0.1), params={"c": Rand(1, 2)}, metadata={"d": 4}, cost=2.0, )) assert trial is report.trial report = cloudpickle.loads(cloudpickle.dumps(report)) assert 0.1 == report.metric assert type(report.metric) == float assert {"c": Rand(1, 2)} == report.params assert {"d": 4} == report.metadata assert 2.0 == report.cost assert 0 == report.rung assert 0.1 == report.sort_metric report = copy.deepcopy( TrialReport(trial, metric=np.float(0.111), cost=2.0, rung=4, sort_metric=1.23)) assert trial is report.trial report = cloudpickle.loads(cloudpickle.dumps(report)) assert 0.111 == report.metric assert type(report.metric) == float assert {"a": Rand(3, 4)} == report.params assert {} == report.metadata assert 2.0 == report.cost assert 4 == report.rung r1 = report.generate_sort_metric(True, 2) r2 = report.generate_sort_metric(False, 1) r3 = report.with_sort_metric(0.234) assert 1.23 == report.sort_metric assert 0.11 == r1.sort_metric assert -0.1 == r2.sort_metric assert 0.234 == r3.sort_metric ts = report.log_time sleep(0.1) report = cloudpickle.loads(cloudpickle.dumps(report)) nr = report.reset_log_time() assert nr.log_time > report.log_time assert report.log_time == ts assert trial.trial_id == report.trial_id assert 0.111 == report.metric assert type(report.metric) == float assert {"a": Rand(3, 4)} == report.params assert {} == report.metadata assert 2.0 == report.cost assert 3.0 == report.with_cost(3.0).cost assert 5 == report.with_rung(5).rung td = trial.with_dfs({"a": pd.DataFrame}) report = TrialReport(td, metric=np.float(0.1)) assert 0 == len(report.trial.dfs)
def rp(tid, metric, rung=0, keys=[]): t = Trial(tid, {}, keys=keys) return TrialReport(t, metric=metric, rung=rung)