def test_suggest(tmpdir):
TUNE_OBJECT_FACTORY.set_temp_path(str(tmpdir))
data = load_diabetes(as_frame=True)
alldata = data["data"]
alldata["label"] = data["target"]
alldata = alldata.sample(frac=1, random_state=0)
test = alldata.iloc[:20]
train = alldata.iloc[20:]
space1 = sk_space(LinearRegression, fit_intercept=Grid(True, False))
space2 = sk_space(Lasso,
fit_intercept=Grid(True, False),
alpha=Rand(0.5, 1)).sample(3, 0)
space3 = sk_space(Lasso,
fit_intercept=Grid(True, False),
alpha=Rand(0.5, 1))
result = suggest_sk_models(
space1 + space3,
train,
test,
"neg_mean_absolute_error",
top_n=0,
distributed=False,
objective_runner=HyperoptRunner(max_iter=10, seed=0),
)
# print([x.jsondict for x in result])
assert 4 == len(result)
assert 50 > result[0].sort_metric
result = suggest_sk_models(
space1 + space2,
train,
test,
"neg_mean_absolute_error",
top_n=0,
partition_keys=["sex"],
temp_path=str(tmpdir),
save_model=True,
)
assert 16 == len(result)
assert 50 > result[0].sort_metric
result = suggest_sk_models(
space1 + space2,
train,
test,
"neg_mean_absolute_error",
top_n=1,
partition_keys=["sex"],
execution_engine=DaskExecutionEngine,
)
assert 2 == len(result)
assert 50 > result[0].sort_metric
def test_space_with_funct():
s = Space(a=1, b=FuncParam(lambda x, y: x + y, x=Grid(0, 1), y=Grid(3, 4)))
assert [
dict(a=1, b=3),
dict(a=1, b=4),
dict(a=1, b=4),
dict(a=1, b=5),
] == list(s)
u = Grid(0, 1)
s = Space(a=u, b=FuncParam(lambda x, y: x + y, x=u, y=u))
assert [
dict(a=0, b=0),
dict(a=1, b=2),
] == list(s)
def test_sk_space():
rows = list(
sk_space("sklearn.linear_model.LinearRegression",
fit_intercept=Grid(True, False)))
assert 2 == len(rows)
assert "sklearn.linear_model._base.LinearRegression" == rows[
0].simple_value[SPACE_MODEL_NAME]
def test_vertical_space():
spaces = list(VerticalSpace())
assert 0 == len(spaces)
spaces = list(VerticalSpace(dict(a=10)))
assert [dict(a=10)] == spaces
spaces = list(VerticalSpace(dict(a=10), [dict(b=11), dict(c=12)]))
assert [dict(a=10), dict(b=11), dict(c=12)] == spaces
spaces = list(VerticalSpace(HorizontalSpace(a=10), dict(b=10)))
assert [dict(a=10), dict(b=10)] == spaces
raises(ValueError, lambda: VerticalSpace(10))
assert not VerticalSpace(HorizontalSpace(a=10),
dict(b=10)).has_random_parameter
space = VerticalSpace(Space(a=Grid(1, 2), b=Rand(0.0, 1.0)),
Space(c=Choice(3, 4)))
ec = list(space.encode())
assert space.has_random_parameter
assert list(space.sample(5, 0)) == list(space.sample(5, 0))
assert list(space.sample(5, 0)) != list(space.sample(5, 1))
dicts = list(space.sample(5, 0))
assert 15 == len(dicts)
assert 5 == len(set(d["b"] for d in dicts if "b" in d))
assert ec == list(space.encode())
def test_horizontal_space():
spaces = list(HorizontalSpace())
assert 1 == len(spaces)
assert {} == spaces[0]
spaces = list(HorizontalSpace(a=10, b=[1, 2], c=dict(x=1)))
assert 1 == len(spaces)
assert dict(a=10, b=[1, 2], c=dict(x=1)) == spaces[0]
spaces = list(HorizontalSpace(dict(a=10, b=[1, 2], c=dict(x=1))))
assert 1 == len(spaces)
assert dict(a=10, b=[1, 2], c=dict(x=1)) == spaces[0]
spaces = list(HorizontalSpace(dict(a=10), dict(b=[1, 2], c=dict(x=1))))
assert 1 == len(spaces)
assert dict(a=10, b=[1, 2], c=dict(x=1)) == spaces[0]
raises(ValueError, lambda: HorizontalSpace(10))
assert not HorizontalSpace(dict(a=10, b=[1, 2],
c=dict(x=1))).has_random_parameter
space = HorizontalSpace(Space(a=Grid(1, 2), b=Rand(0.0, 1.0)),
Space(c=Choice(3, 4)))
ec = list(space.encode())
assert space.has_random_parameter
assert list(space.sample(5, 0)) == list(space.sample(5, 0))
assert list(space.sample(5, 0)) != list(space.sample(5, 1))
dicts = list(space.sample(5, 0))
assert 10 == len(dicts)
assert 5 == len(set(d["b"] for d in dicts))
assert ec == list(space.encode())
def test_suggest_cv(tmpdir):
TUNE_OBJECT_FACTORY.set_temp_path(str(tmpdir))
data = load_diabetes(as_frame=True)
train = data["data"]
train["label"] = data["target"]
space1 = sk_space(LinearRegression, fit_intercept=Grid(True, False))
space2 = sk_space(Lasso,
fit_intercept=Grid(True, False),
alpha=Rand(0.5, 1)).sample(3, 0)
space3 = sk_space(Lasso,
fit_intercept=Grid(True, False),
alpha=Rand(0.5, 1))
result = suggest_sk_models_by_cv(
space1 + space3,
train,
"neg_mean_absolute_error",
top_n=0,
distributed=False,
local_optimizer=HyperoptLocalOptimizer(max_iter=10, seed=0),
)
assert 4 == len(result)
assert 50 > result[0].sort_metric
result = suggest_sk_models_by_cv(
space1 + space2,
train,
"neg_mean_absolute_error",
top_n=0,
partition_keys=["sex"],
temp_path=str(tmpdir),
save_model=True,
)
assert 16 == len(result)
assert 50 > result[0].sort_metric
result = suggest_sk_models_by_cv(
space1 + space2,
train,
"neg_mean_absolute_error",
top_n=1,
partition_keys=["sex"],
execution_engine=DaskExecutionEngine,
)
assert 2 == len(result)
assert 50 > result[0].sort_metric
def test_single_space_sample():
assert not Space(a=1).has_stochastic
assert not Space(a=1, b=Grid(1, 2)).has_stochastic
assert Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)]).has_stochastic
dicts = list(Space(a=1, b=Grid(1, 2)).sample(100))
assert 2 == len(dicts)
dicts = list(Space(a=1, b=RandInt(1, 2)).sample(100))
assert 100 == len(dicts)
space = Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)], c=Choice("a", "b"))
assert list(space.sample(5, 0)) == list(space.sample(5, 0))
assert list(space.sample(5, 0)) != list(space.sample(5, 1))
dicts = list(space.sample(5, 0))
assert 10 == len(dicts)
assert 5 == len(set(d.template["b"][1] for d in dicts))
def test_single_space_sample():
assert not Space(a=1).has_random_parameter
assert not Space(a=1, b=Grid(1, 2)).has_random_parameter
assert Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)]).has_random_parameter
dicts = list(Space(a=1, b=Grid(1, 2)).sample(100))
assert 2 == len(dicts)
space = Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)], c=Choice("a", "b"))
ec = list(space.encode())
assert list(space.sample(5, 0)) == list(space.sample(5, 0))
assert list(space.sample(5, 0)) != list(space.sample(5, 1))
dicts = list(space.sample(5, 0))
assert 10 == len(dicts)
assert 5 == len(set(d["b"][1] for d in dicts))
assert ec == list(
space.encode()) # make sure the instance value was not altered
def test_single_space():
raises(ValueError, lambda: Space("abc"))
raises(ValueError, lambda: Space(1))
raises(ValueError, lambda: Space(1, 2))
space = Space(a=1, b=Grid(2, 3, 4))
dicts = list(space)
dicts = list(space) # make sure iteration doesn't alter the object
assert 3 == len(dicts)
assert dict(a=1, b=2) == dicts[0]
assert dict(a=1, b=3) == dicts[1]
dicts = list(Space(dict(a=Grid(None, "x"), b=Grid(2, 3))))
assert 4 == len(dicts)
dicts = list(
Space(TuningParametersTemplate(dict(a=1, b=[Grid(2, 3),
Grid(4, 5)]))))
assert 4 == len(dicts)
assert dict(a=1, b=[2, 4]) == dicts[0]
assert dict(a=1, b=[2, 5]) == dicts[1]
assert dict(a=1, b=[3, 4]) == dicts[2]
assert dict(a=1, b=[3, 5]) == dicts[3]
dicts = list(Space(a=1, b=dict(x=Grid(2, 3), y=Grid(4, 5))))
assert 4 == len(dicts)
assert dict(a=1, b=dict(x=2, y=4)) == dicts[0]
assert dict(a=1, b=dict(x=2, y=5)) == dicts[1]
assert dict(a=1, b=dict(x=3, y=4)) == dicts[2]
assert dict(a=1, b=dict(x=3, y=5)) == dicts[3]
def test_operators():
s1 = Space(a=1, b=Grid(2, 3))
s2 = Space(c=Grid("a", "b"))
assert [
dict(a=1, b=2, c="a"),
dict(a=1, b=2, c="b"),
dict(a=1, b=3, c="a"),
dict(a=1, b=3, c="b"),
] == list(s1 * s2)
assert [
dict(a=1, b=2),
dict(a=1, b=3),
dict(c="a"),
dict(c="b"),
] == list(s1 + s2)
assert [
dict(a=1, b=2, c="a"),
dict(a=1, b=2, c="b"),
dict(a=1, b=3, c="a"),
dict(a=1, b=3, c="b"),
] == list(s1 * (Space(c="a") + Space(c="b")))
assert [
dict(a=1, b=2),
dict(a=1, b=3),
dict(c="a"),
dict(c="b"),
] == list(s1 + (Space(c="a") + Space(c="b")))
s1 = Space(a=1, b=Grid(2, 3))
s2 = Space(c=Grid("a", "b"))
s3 = Space(d=5)
assert ([
dict(a=1, b=2),
dict(a=1, b=3),
dict(c="a"),
dict(c="b"),
dict(d=5),
] == list(sum([s1, s2, s3])) == list(sum([s1, s2, s3], None)))
def test_single_space():
space = Space(a=1, b=Grid(2, 3, 4))
dicts = list(space)
dicts = list(space) # make sure iteration doesn't alter the object
assert 3 == len(dicts)
assert dict(a=1, b=2) == dicts[0]
assert dict(a=1, b=3) == dicts[1]
dicts = list(Space(a=Grid(None, "x"), b=Grid(2, 3)))
assert 4 == len(dicts)
dicts = list(Space(a=1, b=[Grid(2, 3), Grid(4, 5)]))
assert 4 == len(dicts)
assert dict(a=1, b=[2, 4]) == dicts[0]
assert dict(a=1, b=[2, 5]) == dicts[1]
assert dict(a=1, b=[3, 4]) == dicts[2]
assert dict(a=1, b=[3, 5]) == dicts[3]
dicts = list(Space(a=1, b=dict(x=Grid(2, 3), y=Grid(4, 5))))
assert 4 == len(dicts)
assert dict(a=1, b=dict(x=2, y=4)) == dicts[0]
assert dict(a=1, b=dict(x=2, y=5)) == dicts[1]
assert dict(a=1, b=dict(x=3, y=4)) == dicts[2]
assert dict(a=1, b=dict(x=3, y=5)) == dicts[3]