def test_study(tmpdir):
space = Space(a=Grid(-2, 0, 1))
input_df = pd.DataFrame([[0, 1], [1, 1], [0, 2]], columns=["a", "b"])
dag = FugueWorkflow()
monitor = M()
# no data partition
builder = TuneDatasetBuilder(space,
str(tmpdir)).add_df("b", dag.df(input_df))
dataset = builder.build(dag, 1)
for distributed in [True, False, None]:
# min_better = True
result = optimize_noniterative(
objective=to_noniterative_objective(objective),
dataset=dataset,
distributed=distributed,
)
result.result()[[TUNE_REPORT, TUNE_REPORT_METRIC
]].output(assert_metric,
params=dict(metrics=[3.0, 4.0, 7.0]))
result.result(2)[[TUNE_REPORT, TUNE_REPORT_METRIC
]].output(assert_metric,
params=dict(metrics=[3.0, 4.0]))
# min_better = False
result = optimize_noniterative(
objective=to_noniterative_objective(objective, min_better=False),
dataset=dataset,
distributed=distributed,
)
result.result()[[TUNE_REPORT, TUNE_REPORT_METRIC
]].output(assert_metric,
params=dict(metrics=[-7.0, -4.0, -3.0]))
result.result(2)[[TUNE_REPORT, TUNE_REPORT_METRIC
]].output(assert_metric,
params=dict(metrics=[-7.0, -4.0]))
# with data partition
builder = TuneDatasetBuilder(space, str(tmpdir)).add_df(
"b",
dag.df(input_df).partition_by("a"))
dataset = builder.build(dag, 1)
for distributed in [True, False, None]:
result = optimize_noniterative(
objective=to_noniterative_objective(objective),
dataset=dataset,
distributed=distributed,
monitor=monitor,
)
result.result()[[TUNE_REPORT, TUNE_REPORT_METRIC]].output(
assert_metric, params=dict(metrics=[2.0, 3.0, 6.0, 1.0, 2.0, 5.0]))
result.result(1)[[TUNE_REPORT, TUNE_REPORT_METRIC
]].output(assert_metric,
params=dict(metrics=[1.0, 2.0]))
dag.run()
assert 3 * 3 * 2 == len(monitor._reports)
def test_noniterative_objective():
@noniterative_objective
def f1(a, b: int) -> float:
return a - b
assert isinstance(f1, NonIterativeObjectiveFunc)
trial = Trial("abc", dict(b=20, a=10), dict(c=3))
report = f1.safe_run(trial)
assert report.trial is trial
assert report.metric == -10
assert report.params == trial.params
assert report.metadata == {}
func = to_noniterative_objective("f1")
report = func.safe_run(trial)
assert report.trial is trial
assert report.metric == -10
assert report.params == trial.params
assert report.metadata == {}
@noniterative_objective(min_better=False)
def f2(a, b: int) -> float:
return a - b
assert isinstance(f2, NonIterativeObjectiveFunc)
assert not f2.min_better
def _object_to_noniterative_objective(
self, obj: Any) -> NonIterativeObjectiveFunc:
assert_or_throw(obj is not None,
TuneCompileError("objective can't be None"))
if isinstance(obj, NonIterativeObjectiveFunc):
return obj
if callable(obj):
return to_noniterative_objective(obj)
raise TuneCompileError(
f"{obj} can't be converted to non iterative objective function")
def test_to_noniterative_objective():
def f1(a, b: int) -> float:
return a - b
func = to_noniterative_objective(f1)
assert func.min_better
trial = Trial("abc", dict(b=20, a=10), dict(c=3))
report = func.run(trial)
assert report.trial is trial
assert report.metric == -10
assert report.params == trial.params
assert report.metadata == {}
func = to_noniterative_objective("f1", min_better=False)
assert not func.min_better
trial = Trial("abc", dict(b=20, a=10), dict(c=3))
report = func.run(trial)
assert report.trial is trial
assert report.metric == -10
assert report.params == trial.params
assert report.metadata == {}
assert -1 == func(1, 2)
def f2(a, b: int) -> Tuple[float, Dict[str, Any]]:
return a - b, dict(c=5)
func = to_noniterative_objective(f2)
trial = Trial("abc", dict(b=20, a=10), dict(c=3))
report = func.run(trial)
assert report.trial is trial
assert report.metric == -10
assert report.params == trial.params
assert report.metadata == dict(c=5)
def f3(t: Trial) -> TrialReport:
return TrialReport(
t, t.params["a"] - t.params["b"], params=dict(a=1), metadata=dict(c=6)
)
func = to_noniterative_objective(f3)
trial = Trial("abc", dict(b=20, a=10), dict(c=3))
report = func.run(trial)
assert report.trial is trial
assert report.metric == -10
assert report.params == dict(a=1)
assert report.metadata == dict(c=6)
class F4(NonIterativeObjectiveFunc):
def run(self, t: Trial) -> TrialReport:
return TrialReport(
t, t.params["a"] - t.params["b"], params=dict(a=1), metadata=dict(c=6)
)
f4 = F4()
f4_ = to_noniterative_objective(f4)
assert isinstance(f4_, F4)
assert f4 is not f4_
raises(TuneCompileError, lambda: to_noniterative_objective("abc"))
def test_noniterative_objective():
factory = TuneObjectFactory()
assert isinstance(
factory.make_noniterative_objective(_nobjective),
NonIterativeObjectiveFunc,
)
with raises(TuneCompileError):
factory.make_noniterative_objective("x")
factory.set_noniterative_objective_converter(
lambda x: to_noniterative_objective(_nobjective))
assert isinstance(
factory.make_noniterative_objective("x"),
NonIterativeObjectiveFunc,
)