def f3(t: Trial) -> TrialReport: return TrialReport( t, t.params.simple_value["a"] - t.params.simple_value["b"], params=dict(a=1), metadata=dict(c=6), )
def test_report_logger(): class Mock(TrialReportLogger): def __init__(self, new_best_only: bool = False): super().__init__(new_best_only=new_best_only) self._values = [] def log(self, report: TrialReport) -> None: return self._values.append(report) @property def records(self): return self._values t1 = Trial("a", dict(a=1, b=2), keys=["x", "y"]) r1 = TrialReport(t1, 0.8, sort_metric=-0.8) t2 = Trial("b", dict(a=11, b=12), keys=["xx", "y"]) r2 = TrialReport(t2, 0.7, sort_metric=-0.7) t3 = Trial("c", dict(a=10, b=20), keys=["x", "y"]) r3 = TrialReport(t3, 0.9, sort_metric=-0.9) b = Mock() assert 0 == len(b.records) assert b.best is None assert b.on_report(r1) assert b.on_report(r3) assert r3 is b.best assert 2 == len(b.records) b = Mock() assert b.on_report(r3) assert not b.on_report(r1) assert r3 is b.best assert 2 == len(b.records) b = Mock(new_best_only=True) assert b.on_report(r3) assert not b.on_report(r1) assert r3 is b.best assert 1 == len(b.records)
def test_trial_decision(): trial = Trial("abc", {"a": 1}, {"b": Rand(0, 2)}) report = TrialReport(trial, metric=np.float(0.1), params={"c": Rand(0, 3)}, metadata={"d": 4}) decision = TrialDecision(report, budget=0.0, should_checkpoint=True, metadata={"x": 1}, reason="p") assert trial is decision.trial assert report is decision.report decision = cloudpickle.loads(cloudpickle.dumps(decision)) assert decision.should_stop assert decision.should_checkpoint assert {"x": 1} == decision.metadata assert "p" == decision.reason assert 0.0 == decision.budget assert copy.copy(decision) is decision assert copy.deepcopy(decision) is decision d2 = cloudpickle.loads(cloudpickle.dumps(decision)) assert d2.trial_id == trial.trial_id # assert Rand(0, 3) == d2.report.params["c"] assert decision.should_stop assert decision.should_checkpoint assert {"x": 1} == decision.metadata assert "p" == decision.reason decision = TrialDecision(report, budget=1.0, should_checkpoint=True, metadata={"x": 1}) assert 1.0 == decision.budget assert not decision.should_stop print(decision)
def run(self, t: Trial) -> TrialReport: return TrialReport(t, t.params["a"] - t.params["b"], params=dict(a=1), metadata=dict(c=6))
def to_report(self, v: Any, trial: Trial) -> TrialReport: return TrialReport(trial, metric=float(v[0]), params=trial.params, metadata=v[1])
def test_trial_report(): trial = Trial("abc", {"a": Rand(3, 4)}, {"b": 2}) report = copy.copy( TrialReport( trial, metric=np.float(0.1), params={"c": Rand(1, 2)}, metadata={"d": 4}, cost=2.0, )) assert trial is report.trial report = cloudpickle.loads(cloudpickle.dumps(report)) assert 0.1 == report.metric assert type(report.metric) == float assert {"c": Rand(1, 2)} == report.params assert {"d": 4} == report.metadata assert 2.0 == report.cost assert 0 == report.rung assert 0.1 == report.sort_metric report = copy.deepcopy( TrialReport(trial, metric=np.float(0.111), cost=2.0, rung=4, sort_metric=1.23)) assert trial is report.trial report = cloudpickle.loads(cloudpickle.dumps(report)) assert 0.111 == report.metric assert type(report.metric) == float assert {"a": Rand(3, 4)} == report.params assert {} == report.metadata assert 2.0 == report.cost assert 4 == report.rung r1 = report.generate_sort_metric(True, 2) r2 = report.generate_sort_metric(False, 1) r3 = report.with_sort_metric(0.234) assert 1.23 == report.sort_metric assert 0.11 == r1.sort_metric assert -0.1 == r2.sort_metric assert 0.234 == r3.sort_metric ts = report.log_time sleep(0.1) report = cloudpickle.loads(cloudpickle.dumps(report)) nr = report.reset_log_time() assert nr.log_time > report.log_time assert report.log_time == ts assert trial.trial_id == report.trial_id assert 0.111 == report.metric assert type(report.metric) == float assert {"a": Rand(3, 4)} == report.params assert {} == report.metadata assert 2.0 == report.cost assert 3.0 == report.with_cost(3.0).cost assert 5 == report.with_rung(5).rung td = trial.with_dfs({"a": pd.DataFrame}) report = TrialReport(td, metric=np.float(0.1)) assert 0 == len(report.trial.dfs)
def test_trial_report_heap(): t1 = Trial("a", {}) r1 = TrialReport(t1, 0.1) t2 = Trial("b", {}) r2 = TrialReport(t2, 0.2) t3 = Trial("c", {}) r3 = TrialReport(t3, 0.3) r4 = TrialReport(t3, -0.3) h = TrialReportHeap(min_heap=True) for r in [r1, r2, r3, r4]: h.push(r) assert "a" in h assert "x" not in h for r in [r4, r1, r2]: assert h.pop() is r assert 0 == len(h) h = TrialReportHeap(min_heap=False) for r in [r1, r2, r3, r4]: h.push(r) for r in [r1, r2, r4]: assert r in list(h.values()) for r in [r2, r1, r4]: assert h.pop() is r assert 0 == len(h) # test __lt__, the sort key is sort_metric! r5 = TrialReport(t1, metric=0.1, sort_metric=-0.1) r6 = TrialReport(t2, metric=0.2, sort_metric=-0.2) r7 = TrialReport(t3, metric=0.3, sort_metric=-0.3) h = TrialReportHeap(min_heap=True) for r in [r7, r6, r5]: h.push(r) for r in [r7, r6, r5]: assert h.pop() is r assert 0 == len(h) r5 = TrialReport(t1, metric=0.1, cost=0.2, rung=5) r6 = TrialReport(t2, metric=0.1, cost=0.3, rung=5) r7 = TrialReport(t3, metric=0.1, cost=0.3, rung=6) h = TrialReportHeap(min_heap=True) for r in [r7, r6, r5]: h.push(r) for r in [r5, r6, r7]: assert h.pop() is r assert 0 == len(h) # equal case r8 = TrialReport(t1, metric=0.1, cost=0.3, rung=6) r9 = TrialReport(t2, metric=0.1, cost=0.3, rung=6) h = TrialReportHeap(min_heap=False) for r in [r8, r9]: h.push(r) for r in [r8, r9]: assert h.pop() is r assert 0 == len(h)
def run_single_iteration(self): trial = self.current_trial metric = self._all[trial.params.simple_value["a"]][self._it] self._it += 1 return TrialReport(trial, metric=metric)
def rp(tid, metric, rung=0, keys=[]): t = Trial(tid, {}, keys=keys) return TrialReport(t, metric=metric, rung=rung)
def log(self, report: TrialReport) -> None: self._reports.append(report.reset_log_time())
def run_single_iteration(self) -> TrialReport: self.v += 1 return TrialReport(self.current_trial, self.v, metadata={"d": 4})