def test_hyperband(tmpdir):
TUNE_OBJECT_FACTORY.set_temp_path(str(tmpdir))
space = keras_space(MockSpec, l1=RandInt(8, 16), l2=RandInt(8, 24))
with raises(TuneCompileError):
suggest_keras_models_by_hyperband(
space,
plans=[
[(2.0, 4), (4.0, 2)],
[(4.0, 2), (2.0, 4)],
],
)
space = space.sample(10, 0)
reports = suggest_keras_models_by_hyperband(
space,
plans=[
[(2.0, 4), (4.0, 2)],
[(4.0, 2), (2.0, 4)],
],
top_n=2,
)
for r in reports:
print(r)
assert 2 == len(reports)
def test_randint(self):
for log in [True, False]:
# common case
values = self._generate_values(RandInt(1, 3, log=log),
lambda x: x**2)
assert_close(values, [1, 2, 3])
assert all(isinstance(x, int) for x in values)
values = self._generate_values(
RandInt(1, 3, include_high=False, log=log), lambda x: x**2)
assert_close(values, [1, 2])
# with q, range % q != 0
for ih in [True, False]:
values = self._generate_values(
RandInt(1, 6, 2, include_high=ih, log=log),
lambda x: x**2)
assert_close(values, [1, 3, 5])
assert all(isinstance(x, int) for x in values)
# with q, range % q == 0
values = self._generate_values(RandInt(1, 5, 2, log=log),
lambda x: x**2)
assert_close(values, [1, 3, 5])
values = self._generate_values(
RandInt(1, 5, 2, include_high=False, log=log),
lambda x: x**2)
assert_close(values, [1, 3])
def test_sha(tmpdir):
TUNE_OBJECT_FACTORY.set_temp_path(str(tmpdir))
space = keras_space(MockSpec, l1=RandInt(8, 16), l2=RandInt(8, 24))
with raises(TuneCompileError):
suggest_keras_models_by_sha(space, plan=[(2.0, 4), (4.0, 2)])
space = space.sample(6, 0)
reports = suggest_keras_models_by_sha(space, plan=[(2.0, 4), (4.0, 2)], top_n=2)
for r in reports:
print(r.jsondict)
assert 2 == len(reports)
def test_objective(tmpdir):
dfs = load_iris(as_frame=True)
df = dfs["data"]
df["label"] = dfs["target"]
df = df[df.label <= 1]
t = Trial(
"x",
params={
"max_iter": RandInt(2, 4),
SPACE_MODEL_NAME: "sklearn.linear_model.LogisticRegression",
},
dfs={TUNE_DATASET_DF_DEFAULT_NAME: df},
)
obj = SKCVObjective(scoring="accuracy")
runner = HyperoptRunner(5, 0)
def v(report):
print(report.jsondict)
assert report.sort_metric < 0
assert "cv_scores" in report.metadata
# assert report.trial.params["max_iter"] >= 2
validate_noniterative_objective(obj, t, v, runner=runner)
obj = SKCVObjective(scoring="accuracy", checkpoint_path=str(tmpdir))
def v2(report):
print(report.jsondict)
assert report.sort_metric < 0
assert "cv_scores" in report.metadata
assert "checkpoint_path" in report.metadata
# assert report.trial.params["max_iter"] >= 2
validate_noniterative_objective(obj, t, v2, runner=runner)
def test_hyperopt():
params = dict(a=Rand(-10.0, 10.0), b=RandInt(-100, 100), c=2.0)
trial = Trial("a", params, metadata={})
h = HyperoptRunner(max_iter=200, seed=0)
@noniterative_objective
def objective(a, b, c) -> Tuple[float, Dict[str, Any]]:
return a**2 + b**2 + c, dict(a=1)
def v(report):
assert report.metric < 7
assert report.params["a"]**2 < 2
assert report.params["b"]**2 < 2
assert 2.0 == report.params["c"]
validate_noniterative_objective(objective, trial, v, runner=h)
def test_single_space_sample():
assert not Space(a=1).has_stochastic
assert not Space(a=1, b=Grid(1, 2)).has_stochastic
assert Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)]).has_stochastic
dicts = list(Space(a=1, b=Grid(1, 2)).sample(100))
assert 2 == len(dicts)
dicts = list(Space(a=1, b=RandInt(1, 2)).sample(100))
assert 100 == len(dicts)
space = Space(a=1, b=[Grid(1, 2), Rand(0.0, 1.0)], c=Choice("a", "b"))
assert list(space.sample(5, 0)) == list(space.sample(5, 0))
assert list(space.sample(5, 0)) != list(space.sample(5, 1))
dicts = list(space.sample(5, 0))
assert 10 == len(dicts)
assert 5 == len(set(d.template["b"][1] for d in dicts))
def test_optimization(self):
params = dict(a=Rand(-10.0, 10.0), b=RandInt(-100, 100), c=2.0)
trial = Trial("a", params, metadata={})
o = self.make_optimizer(max_iter=200)
@noniterative_objective
def objective(a, b, c) -> Tuple[float, Dict[str, Any]]:
return a**2 + b**2 + c, dict(a="x")
def v(report):
print(report.metric)
assert report.metric < 7
assert report.params.simple_value["a"]**2 < 2
assert report.params.simple_value["b"]**2 < 2
assert 2.0 == report.params.simple_value["c"]
assert "x" == report.metadata["a"]
validate_noniterative_objective(objective, trial, v, optimizer=o)