def test_vertical_space():
spaces = list(VerticalSpace())
assert 0 == len(spaces)
spaces = list(VerticalSpace(dict(a=10)))
assert [dict(a=10)] == spaces
spaces = list(VerticalSpace(dict(a=10), [dict(b=11), dict(c=12)]))
assert [dict(a=10), dict(b=11), dict(c=12)] == spaces
spaces = list(VerticalSpace(HorizontalSpace(a=10), dict(b=10)))
assert [dict(a=10), dict(b=10)] == spaces
raises(ValueError, lambda: VerticalSpace(10))
assert not VerticalSpace(HorizontalSpace(a=10),
dict(b=10)).has_random_parameter
space = VerticalSpace(Space(a=Grid(1, 2), b=Rand(0.0, 1.0)),
Space(c=Choice(3, 4)))
ec = list(space.encode())
assert space.has_random_parameter
assert list(space.sample(5, 0)) == list(space.sample(5, 0))
assert list(space.sample(5, 0)) != list(space.sample(5, 1))
dicts = list(space.sample(5, 0))
assert 15 == len(dicts)
assert 5 == len(set(d["b"] for d in dicts if "b" in d))
assert ec == list(space.encode())
def test_horizontal_space():
spaces = list(HorizontalSpace())
assert 1 == len(spaces)
assert {} == spaces[0]
spaces = list(HorizontalSpace(a=10, b=[1, 2], c=dict(x=1)))
assert 1 == len(spaces)
assert dict(a=10, b=[1, 2], c=dict(x=1)) == spaces[0]
spaces = list(HorizontalSpace(dict(a=10, b=[1, 2], c=dict(x=1))))
assert 1 == len(spaces)
assert dict(a=10, b=[1, 2], c=dict(x=1)) == spaces[0]
spaces = list(HorizontalSpace(dict(a=10), dict(b=[1, 2], c=dict(x=1))))
assert 1 == len(spaces)
assert dict(a=10, b=[1, 2], c=dict(x=1)) == spaces[0]
raises(ValueError, lambda: HorizontalSpace(10))
assert not HorizontalSpace(dict(a=10, b=[1, 2],
c=dict(x=1))).has_random_parameter
space = HorizontalSpace(Space(a=Grid(1, 2), b=Rand(0.0, 1.0)),
Space(c=Choice(3, 4)))
ec = list(space.encode())
assert space.has_random_parameter
assert list(space.sample(5, 0)) == list(space.sample(5, 0))
assert list(space.sample(5, 0)) != list(space.sample(5, 1))
dicts = list(space.sample(5, 0))
assert 10 == len(dicts)
assert 5 == len(set(d["b"] for d in dicts))
assert ec == list(space.encode())
def test_space_with_funct():
s = Space(a=1, b=FuncParam(lambda x, y: x + y, x=Grid(0, 1), y=Grid(3, 4)))
assert [
dict(a=1, b=3),
dict(a=1, b=4),
dict(a=1, b=4),
dict(a=1, b=5),
] == list(s)
u = Grid(0, 1)
s = Space(a=u, b=FuncParam(lambda x, y: x + y, x=u, y=u))
assert [
dict(a=0, b=0),
dict(a=1, b=2),
] == list(s)
def test_single_space_sample():
assert not Space(a=1).has_stochastic
assert not Space(a=1, b=Grid(1, 2)).has_stochastic
assert Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)]).has_stochastic
dicts = list(Space(a=1, b=Grid(1, 2)).sample(100))
assert 2 == len(dicts)
dicts = list(Space(a=1, b=RandInt(1, 2)).sample(100))
assert 100 == len(dicts)
space = Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)], c=Choice("a", "b"))
assert list(space.sample(5, 0)) == list(space.sample(5, 0))
assert list(space.sample(5, 0)) != list(space.sample(5, 1))
dicts = list(space.sample(5, 0))
assert 10 == len(dicts)
assert 5 == len(set(d.template["b"][1] for d in dicts))
def test_single_space():
raises(ValueError, lambda: Space("abc"))
raises(ValueError, lambda: Space(1))
raises(ValueError, lambda: Space(1, 2))
space = Space(a=1, b=Grid(2, 3, 4))
dicts = list(space)
dicts = list(space) # make sure iteration doesn't alter the object
assert 3 == len(dicts)
assert dict(a=1, b=2) == dicts[0]
assert dict(a=1, b=3) == dicts[1]
dicts = list(Space(dict(a=Grid(None, "x"), b=Grid(2, 3))))
assert 4 == len(dicts)
dicts = list(
Space(TuningParametersTemplate(dict(a=1, b=[Grid(2, 3),
Grid(4, 5)]))))
assert 4 == len(dicts)
assert dict(a=1, b=[2, 4]) == dicts[0]
assert dict(a=1, b=[2, 5]) == dicts[1]
assert dict(a=1, b=[3, 4]) == dicts[2]
assert dict(a=1, b=[3, 5]) == dicts[3]
dicts = list(Space(a=1, b=dict(x=Grid(2, 3), y=Grid(4, 5))))
assert 4 == len(dicts)
assert dict(a=1, b=dict(x=2, y=4)) == dicts[0]
assert dict(a=1, b=dict(x=2, y=5)) == dicts[1]
assert dict(a=1, b=dict(x=3, y=4)) == dicts[2]
assert dict(a=1, b=dict(x=3, y=5)) == dicts[3]
def test_dataset(tmpdir):
factory = TuneObjectFactory()
factory.set_temp_path(str(tmpdir))
df = pd.DataFrame([[0]], columns=["a"])
dag = FugueWorkflow()
data = factory.make_dataset(dag, Space(a=1, b=1), df=df)
assert isinstance(data, TuneDataset)
assert factory.make_dataset(dag, data) is data
# TODO: the unit test is not complete, but it's covered by other functions
with raises(TuneCompileError):
factory.make_dataset(dag, 1)
def test_operators():
s1 = Space(a=1, b=Grid(2, 3))
s2 = Space(c=Grid("a", "b"))
assert [
dict(a=1, b=2, c="a"),
dict(a=1, b=2, c="b"),
dict(a=1, b=3, c="a"),
dict(a=1, b=3, c="b"),
] == list(s1 * s2)
assert [
dict(a=1, b=2),
dict(a=1, b=3),
dict(c="a"),
dict(c="b"),
] == list(s1 + s2)
assert [
dict(a=1, b=2, c="a"),
dict(a=1, b=3, c="a"),
dict(a=1, b=2, c="b"),
dict(a=1, b=3, c="b"),
] == list(s1 * [dict(c="a"), dict(c="b")])
assert [
dict(a=1, b=2),
dict(a=1, b=3),
dict(c="a"),
dict(c="b"),
] == list(s1 + [dict(c="a"), dict(c="b")])
s1 = Space(a=1, b=Grid(2, 3))
s2 = Space(c=Grid("a", "b"))
s3 = Space(d=5)
assert ([
dict(a=1, b=2),
dict(a=1, b=3),
dict(c="a"),
dict(c="b"),
dict(d=5),
] == list(sum([s1, s2, s3])) == list(sum([s1, s2, s3], None)))
def test_single_space():
space = Space(a=1, b=Grid(2, 3, 4))
dicts = list(space)
dicts = list(space) # make sure iteration doesn't alter the object
assert 3 == len(dicts)
assert dict(a=1, b=2) == dicts[0]
assert dict(a=1, b=3) == dicts[1]
dicts = list(Space(a=Grid(None, "x"), b=Grid(2, 3)))
assert 4 == len(dicts)
dicts = list(Space(a=1, b=[Grid(2, 3), Grid(4, 5)]))
assert 4 == len(dicts)
assert dict(a=1, b=[2, 4]) == dicts[0]
assert dict(a=1, b=[2, 5]) == dicts[1]
assert dict(a=1, b=[3, 4]) == dicts[2]
assert dict(a=1, b=[3, 5]) == dicts[3]
dicts = list(Space(a=1, b=dict(x=Grid(2, 3), y=Grid(4, 5))))
assert 4 == len(dicts)
assert dict(a=1, b=dict(x=2, y=4)) == dicts[0]
assert dict(a=1, b=dict(x=2, y=5)) == dicts[1]
assert dict(a=1, b=dict(x=3, y=4)) == dicts[2]
assert dict(a=1, b=dict(x=3, y=5)) == dicts[3]
def test_iterative_study(tmpdir):
def assert_metric(df: Iterable[Dict[str, Any]], metric: float) -> None:
for row in df:
assert row[TUNE_REPORT_METRIC] < metric
study = IterativeStudy(F(), str(tmpdir))
space = sum(
Space(a=a, b=b)
for a, b in [(1.1, 0.2), (0.8, -0.2), (1.2, -0.1), (0.7,
0.3), (1.0, 1.5)])
dag = FugueWorkflow()
dataset = TuneDatasetBuilder(space, str(tmpdir)).build(dag)
result = study.optimize(
dataset,
J([1, 2, 3, 4]),
)
result.result(1).show()
result.result(1).output(assert_metric, params=dict(metric=-2.8))
dag.run()
def keras_space(model: Any, **params: Any) -> Space:
expr = to_keras_spec_expr(model)
_TYPE_DICT[expr] = to_keras_spec(model)
data = {SPACE_MODEL_NAME: expr}
data.update(params)
return Space(**data)
def test_single_space_sample():
assert not Space(a=1).has_random_parameter
assert not Space(a=1, b=Grid(1, 2)).has_random_parameter
assert Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)]).has_random_parameter
dicts = list(Space(a=1, b=Grid(1, 2)).sample(100))
assert 2 == len(dicts)
space = Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)], c=Choice("a", "b"))
ec = list(space.encode())
assert list(space.sample(5, 0)) == list(space.sample(5, 0))
assert list(space.sample(5, 0)) != list(space.sample(5, 1))
dicts = list(space.sample(5, 0))
assert 10 == len(dicts)
assert 5 == len(set(d["b"][1] for d in dicts))
assert ec == list(
space.encode()) # make sure the instance value was not altered