def test_suggest(tmpdir): TUNE_OBJECT_FACTORY.set_temp_path(str(tmpdir)) data = load_diabetes(as_frame=True) alldata = data["data"] alldata["label"] = data["target"] alldata = alldata.sample(frac=1, random_state=0) test = alldata.iloc[:20] train = alldata.iloc[20:] space1 = sk_space(LinearRegression, fit_intercept=Grid(True, False)) space2 = sk_space(Lasso, fit_intercept=Grid(True, False), alpha=Rand(0.5, 1)).sample(3, 0) space3 = sk_space(Lasso, fit_intercept=Grid(True, False), alpha=Rand(0.5, 1)) result = suggest_sk_models( space1 + space3, train, test, "neg_mean_absolute_error", top_n=0, distributed=False, objective_runner=HyperoptRunner(max_iter=10, seed=0), ) # print([x.jsondict for x in result]) assert 4 == len(result) assert 50 > result[0].sort_metric result = suggest_sk_models( space1 + space2, train, test, "neg_mean_absolute_error", top_n=0, partition_keys=["sex"], temp_path=str(tmpdir), save_model=True, ) assert 16 == len(result) assert 50 > result[0].sort_metric result = suggest_sk_models( space1 + space2, train, test, "neg_mean_absolute_error", top_n=1, partition_keys=["sex"], execution_engine=DaskExecutionEngine, ) assert 2 == len(result) assert 50 > result[0].sort_metric
def test_vertical_space(): spaces = list(VerticalSpace()) assert 0 == len(spaces) spaces = list(VerticalSpace(dict(a=10))) assert [dict(a=10)] == spaces spaces = list(VerticalSpace(dict(a=10), [dict(b=11), dict(c=12)])) assert [dict(a=10), dict(b=11), dict(c=12)] == spaces spaces = list(VerticalSpace(HorizontalSpace(a=10), dict(b=10))) assert [dict(a=10), dict(b=10)] == spaces raises(ValueError, lambda: VerticalSpace(10)) assert not VerticalSpace(HorizontalSpace(a=10), dict(b=10)).has_random_parameter space = VerticalSpace(Space(a=Grid(1, 2), b=Rand(0.0, 1.0)), Space(c=Choice(3, 4))) ec = list(space.encode()) assert space.has_random_parameter assert list(space.sample(5, 0)) == list(space.sample(5, 0)) assert list(space.sample(5, 0)) != list(space.sample(5, 1)) dicts = list(space.sample(5, 0)) assert 15 == len(dicts) assert 5 == len(set(d["b"] for d in dicts if "b" in d)) assert ec == list(space.encode())
def test_horizontal_space(): spaces = list(HorizontalSpace()) assert 1 == len(spaces) assert {} == spaces[0] spaces = list(HorizontalSpace(a=10, b=[1, 2], c=dict(x=1))) assert 1 == len(spaces) assert dict(a=10, b=[1, 2], c=dict(x=1)) == spaces[0] spaces = list(HorizontalSpace(dict(a=10, b=[1, 2], c=dict(x=1)))) assert 1 == len(spaces) assert dict(a=10, b=[1, 2], c=dict(x=1)) == spaces[0] spaces = list(HorizontalSpace(dict(a=10), dict(b=[1, 2], c=dict(x=1)))) assert 1 == len(spaces) assert dict(a=10, b=[1, 2], c=dict(x=1)) == spaces[0] raises(ValueError, lambda: HorizontalSpace(10)) assert not HorizontalSpace(dict(a=10, b=[1, 2], c=dict(x=1))).has_random_parameter space = HorizontalSpace(Space(a=Grid(1, 2), b=Rand(0.0, 1.0)), Space(c=Choice(3, 4))) ec = list(space.encode()) assert space.has_random_parameter assert list(space.sample(5, 0)) == list(space.sample(5, 0)) assert list(space.sample(5, 0)) != list(space.sample(5, 1)) dicts = list(space.sample(5, 0)) assert 10 == len(dicts) assert 5 == len(set(d["b"] for d in dicts)) assert ec == list(space.encode())
def test_suggest_cv(tmpdir): TUNE_OBJECT_FACTORY.set_temp_path(str(tmpdir)) data = load_diabetes(as_frame=True) train = data["data"] train["label"] = data["target"] space1 = sk_space(LinearRegression, fit_intercept=Grid(True, False)) space2 = sk_space(Lasso, fit_intercept=Grid(True, False), alpha=Rand(0.5, 1)).sample(3, 0) space3 = sk_space(Lasso, fit_intercept=Grid(True, False), alpha=Rand(0.5, 1)) result = suggest_sk_models_by_cv( space1 + space3, train, "neg_mean_absolute_error", top_n=0, distributed=False, local_optimizer=HyperoptLocalOptimizer(max_iter=10, seed=0), ) assert 4 == len(result) assert 50 > result[0].sort_metric result = suggest_sk_models_by_cv( space1 + space2, train, "neg_mean_absolute_error", top_n=0, partition_keys=["sex"], temp_path=str(tmpdir), save_model=True, ) assert 16 == len(result) assert 50 > result[0].sort_metric result = suggest_sk_models_by_cv( space1 + space2, train, "neg_mean_absolute_error", top_n=1, partition_keys=["sex"], execution_engine=DaskExecutionEngine, ) assert 2 == len(result) assert 50 > result[0].sort_metric
def test_single_space_sample(): assert not Space(a=1).has_stochastic assert not Space(a=1, b=Grid(1, 2)).has_stochastic assert Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)]).has_stochastic dicts = list(Space(a=1, b=Grid(1, 2)).sample(100)) assert 2 == len(dicts) dicts = list(Space(a=1, b=RandInt(1, 2)).sample(100)) assert 100 == len(dicts) space = Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)], c=Choice("a", "b")) assert list(space.sample(5, 0)) == list(space.sample(5, 0)) assert list(space.sample(5, 0)) != list(space.sample(5, 1)) dicts = list(space.sample(5, 0)) assert 10 == len(dicts) assert 5 == len(set(d.template["b"][1] for d in dicts))
def test_single_space_sample(): assert not Space(a=1).has_random_parameter assert not Space(a=1, b=Grid(1, 2)).has_random_parameter assert Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)]).has_random_parameter dicts = list(Space(a=1, b=Grid(1, 2)).sample(100)) assert 2 == len(dicts) space = Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)], c=Choice("a", "b")) ec = list(space.encode()) assert list(space.sample(5, 0)) == list(space.sample(5, 0)) assert list(space.sample(5, 0)) != list(space.sample(5, 1)) dicts = list(space.sample(5, 0)) assert 10 == len(dicts) assert 5 == len(set(d["b"][1] for d in dicts)) assert ec == list( space.encode()) # make sure the instance value was not altered
def test_hyperopt(): params = dict(a=Rand(-10.0, 10.0), b=RandInt(-100, 100), c=2.0) trial = Trial("a", params, metadata={}) h = HyperoptRunner(max_iter=200, seed=0) @noniterative_objective def objective(a, b, c) -> Tuple[float, Dict[str, Any]]: return a**2 + b**2 + c, dict(a=1) def v(report): assert report.metric < 7 assert report.params["a"]**2 < 2 assert report.params["b"]**2 < 2 assert 2.0 == report.params["c"] validate_noniterative_objective(objective, trial, v, runner=h)
def test_rand(self): # common case values = self._generate_values(Rand(-2.0, 3.0), lambda x: x**2) assert len(values) > 0 assert all(x >= -2.0 and x <= 3.0 for x in values) # with q, and range%q == 0 values = self._generate_values(Rand(-2.0, 3.0, q=2.5), lambda x: x**2) assert_close(values, [-2.0, 0.5, 3.0]) values = self._generate_values( Rand(-2.0, 3.0, q=2.5, include_high=False), lambda x: x**2) assert_close(values, [-2.0, 0.5]) # with q, and range%q != 0 for ih in [True, False]: values = self._generate_values( Rand(-2.0, 3.0, q=3.0, include_high=ih), lambda x: x**2) assert_close(values, [-2.0, 1.0]) # with log values = self._generate_values(Rand(0.1, 3.0, log=True), lambda x: x**2) assert all(x >= 0.1 and x <= 3.0 for x in values) # with log and q, and range%q == 0 values = self._generate_values(Rand(1.0, 6.0, q=2.5, log=True), lambda x: x**2) assert_close(values, [1.0, 3.5, 6.0]) values = self._generate_values( Rand(1.0, 6.0, q=2.5, log=True, include_high=False), lambda x: x**2) assert_close(values, [1.0, 3.5]) # with log and q, and range%q != 0 for ih in [True, False]: values = self._generate_values( Rand(1.0, 6.0, q=3.0, log=True, include_high=ih), lambda x: x**2) assert_close(values, [1.0, 4.0])
def test_optimization(self): params = dict(a=Rand(-10.0, 10.0), b=RandInt(-100, 100), c=2.0) trial = Trial("a", params, metadata={}) o = self.make_optimizer(max_iter=200) @noniterative_objective def objective(a, b, c) -> Tuple[float, Dict[str, Any]]: return a**2 + b**2 + c, dict(a="x") def v(report): print(report.metric) assert report.metric < 7 assert report.params.simple_value["a"]**2 < 2 assert report.params.simple_value["b"]**2 < 2 assert 2.0 == report.params.simple_value["c"] assert "x" == report.metadata["a"] validate_noniterative_objective(objective, trial, v, optimizer=o)