def test_suggest(tmpdir):
TUNE_OBJECT_FACTORY.set_temp_path(str(tmpdir))
data = load_diabetes(as_frame=True)
alldata = data["data"]
alldata["label"] = data["target"]
alldata = alldata.sample(frac=1, random_state=0)
test = alldata.iloc[:20]
train = alldata.iloc[20:]
space1 = sk_space(LinearRegression, fit_intercept=Grid(True, False))
space2 = sk_space(Lasso,
fit_intercept=Grid(True, False),
alpha=Rand(0.5, 1)).sample(3, 0)
space3 = sk_space(Lasso,
fit_intercept=Grid(True, False),
alpha=Rand(0.5, 1))
result = suggest_sk_models(
space1 + space3,
train,
test,
"neg_mean_absolute_error",
top_n=0,
distributed=False,
objective_runner=HyperoptRunner(max_iter=10, seed=0),
)
# print([x.jsondict for x in result])
assert 4 == len(result)
assert 50 > result[0].sort_metric
result = suggest_sk_models(
space1 + space2,
train,
test,
"neg_mean_absolute_error",
top_n=0,
partition_keys=["sex"],
temp_path=str(tmpdir),
save_model=True,
)
assert 16 == len(result)
assert 50 > result[0].sort_metric
result = suggest_sk_models(
space1 + space2,
train,
test,
"neg_mean_absolute_error",
top_n=1,
partition_keys=["sex"],
execution_engine=DaskExecutionEngine,
)
assert 2 == len(result)
assert 50 > result[0].sort_metric
def test_vertical_space():
spaces = list(VerticalSpace())
assert 0 == len(spaces)
spaces = list(VerticalSpace(dict(a=10)))
assert [dict(a=10)] == spaces
spaces = list(VerticalSpace(dict(a=10), [dict(b=11), dict(c=12)]))
assert [dict(a=10), dict(b=11), dict(c=12)] == spaces
spaces = list(VerticalSpace(HorizontalSpace(a=10), dict(b=10)))
assert [dict(a=10), dict(b=10)] == spaces
raises(ValueError, lambda: VerticalSpace(10))
assert not VerticalSpace(HorizontalSpace(a=10),
dict(b=10)).has_random_parameter
space = VerticalSpace(Space(a=Grid(1, 2), b=Rand(0.0, 1.0)),
Space(c=Choice(3, 4)))
ec = list(space.encode())
assert space.has_random_parameter
assert list(space.sample(5, 0)) == list(space.sample(5, 0))
assert list(space.sample(5, 0)) != list(space.sample(5, 1))
dicts = list(space.sample(5, 0))
assert 15 == len(dicts)
assert 5 == len(set(d["b"] for d in dicts if "b" in d))
assert ec == list(space.encode())
def test_horizontal_space():
spaces = list(HorizontalSpace())
assert 1 == len(spaces)
assert {} == spaces[0]
spaces = list(HorizontalSpace(a=10, b=[1, 2], c=dict(x=1)))
assert 1 == len(spaces)
assert dict(a=10, b=[1, 2], c=dict(x=1)) == spaces[0]
spaces = list(HorizontalSpace(dict(a=10, b=[1, 2], c=dict(x=1))))
assert 1 == len(spaces)
assert dict(a=10, b=[1, 2], c=dict(x=1)) == spaces[0]
spaces = list(HorizontalSpace(dict(a=10), dict(b=[1, 2], c=dict(x=1))))
assert 1 == len(spaces)
assert dict(a=10, b=[1, 2], c=dict(x=1)) == spaces[0]
raises(ValueError, lambda: HorizontalSpace(10))
assert not HorizontalSpace(dict(a=10, b=[1, 2],
c=dict(x=1))).has_random_parameter
space = HorizontalSpace(Space(a=Grid(1, 2), b=Rand(0.0, 1.0)),
Space(c=Choice(3, 4)))
ec = list(space.encode())
assert space.has_random_parameter
assert list(space.sample(5, 0)) == list(space.sample(5, 0))
assert list(space.sample(5, 0)) != list(space.sample(5, 1))
dicts = list(space.sample(5, 0))
assert 10 == len(dicts)
assert 5 == len(set(d["b"] for d in dicts))
assert ec == list(space.encode())
def test_suggest_cv(tmpdir):
TUNE_OBJECT_FACTORY.set_temp_path(str(tmpdir))
data = load_diabetes(as_frame=True)
train = data["data"]
train["label"] = data["target"]
space1 = sk_space(LinearRegression, fit_intercept=Grid(True, False))
space2 = sk_space(Lasso,
fit_intercept=Grid(True, False),
alpha=Rand(0.5, 1)).sample(3, 0)
space3 = sk_space(Lasso,
fit_intercept=Grid(True, False),
alpha=Rand(0.5, 1))
result = suggest_sk_models_by_cv(
space1 + space3,
train,
"neg_mean_absolute_error",
top_n=0,
distributed=False,
local_optimizer=HyperoptLocalOptimizer(max_iter=10, seed=0),
)
assert 4 == len(result)
assert 50 > result[0].sort_metric
result = suggest_sk_models_by_cv(
space1 + space2,
train,
"neg_mean_absolute_error",
top_n=0,
partition_keys=["sex"],
temp_path=str(tmpdir),
save_model=True,
)
assert 16 == len(result)
assert 50 > result[0].sort_metric
result = suggest_sk_models_by_cv(
space1 + space2,
train,
"neg_mean_absolute_error",
top_n=1,
partition_keys=["sex"],
execution_engine=DaskExecutionEngine,
)
assert 2 == len(result)
assert 50 > result[0].sort_metric
def test_single_space_sample():
assert not Space(a=1).has_stochastic
assert not Space(a=1, b=Grid(1, 2)).has_stochastic
assert Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)]).has_stochastic
dicts = list(Space(a=1, b=Grid(1, 2)).sample(100))
assert 2 == len(dicts)
dicts = list(Space(a=1, b=RandInt(1, 2)).sample(100))
assert 100 == len(dicts)
space = Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)], c=Choice("a", "b"))
assert list(space.sample(5, 0)) == list(space.sample(5, 0))
assert list(space.sample(5, 0)) != list(space.sample(5, 1))
dicts = list(space.sample(5, 0))
assert 10 == len(dicts)
assert 5 == len(set(d.template["b"][1] for d in dicts))
def test_single_space_sample():
assert not Space(a=1).has_random_parameter
assert not Space(a=1, b=Grid(1, 2)).has_random_parameter
assert Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)]).has_random_parameter
dicts = list(Space(a=1, b=Grid(1, 2)).sample(100))
assert 2 == len(dicts)
space = Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)], c=Choice("a", "b"))
ec = list(space.encode())
assert list(space.sample(5, 0)) == list(space.sample(5, 0))
assert list(space.sample(5, 0)) != list(space.sample(5, 1))
dicts = list(space.sample(5, 0))
assert 10 == len(dicts)
assert 5 == len(set(d["b"][1] for d in dicts))
assert ec == list(
space.encode()) # make sure the instance value was not altered
def test_hyperopt():
params = dict(a=Rand(-10.0, 10.0), b=RandInt(-100, 100), c=2.0)
trial = Trial("a", params, metadata={})
h = HyperoptRunner(max_iter=200, seed=0)
@noniterative_objective
def objective(a, b, c) -> Tuple[float, Dict[str, Any]]:
return a**2 + b**2 + c, dict(a=1)
def v(report):
assert report.metric < 7
assert report.params["a"]**2 < 2
assert report.params["b"]**2 < 2
assert 2.0 == report.params["c"]
validate_noniterative_objective(objective, trial, v, runner=h)
def test_rand(self):
# common case
values = self._generate_values(Rand(-2.0, 3.0), lambda x: x**2)
assert len(values) > 0
assert all(x >= -2.0 and x <= 3.0 for x in values)
# with q, and range%q == 0
values = self._generate_values(Rand(-2.0, 3.0, q=2.5),
lambda x: x**2)
assert_close(values, [-2.0, 0.5, 3.0])
values = self._generate_values(
Rand(-2.0, 3.0, q=2.5, include_high=False), lambda x: x**2)
assert_close(values, [-2.0, 0.5])
# with q, and range%q != 0
for ih in [True, False]:
values = self._generate_values(
Rand(-2.0, 3.0, q=3.0, include_high=ih), lambda x: x**2)
assert_close(values, [-2.0, 1.0])
# with log
values = self._generate_values(Rand(0.1, 3.0, log=True),
lambda x: x**2)
assert all(x >= 0.1 and x <= 3.0 for x in values)
# with log and q, and range%q == 0
values = self._generate_values(Rand(1.0, 6.0, q=2.5, log=True),
lambda x: x**2)
assert_close(values, [1.0, 3.5, 6.0])
values = self._generate_values(
Rand(1.0, 6.0, q=2.5, log=True, include_high=False),
lambda x: x**2)
assert_close(values, [1.0, 3.5])
# with log and q, and range%q != 0
for ih in [True, False]:
values = self._generate_values(
Rand(1.0, 6.0, q=3.0, log=True, include_high=ih),
lambda x: x**2)
assert_close(values, [1.0, 4.0])
def test_optimization(self):
params = dict(a=Rand(-10.0, 10.0), b=RandInt(-100, 100), c=2.0)
trial = Trial("a", params, metadata={})
o = self.make_optimizer(max_iter=200)
@noniterative_objective
def objective(a, b, c) -> Tuple[float, Dict[str, Any]]:
return a**2 + b**2 + c, dict(a="x")
def v(report):
print(report.metric)
assert report.metric < 7
assert report.params.simple_value["a"]**2 < 2
assert report.params.simple_value["b"]**2 < 2
assert 2.0 == report.params.simple_value["c"]
assert "x" == report.metadata["a"]
validate_noniterative_objective(objective, trial, v, optimizer=o)