def test_hyperopt_algorithm(self):
config = V1Hyperopt.from_dict(
{
"concurrency": 2,
"algorithm": "rand",
"numRuns": 1,
"params": {"param": {"kind": "uniform", "value": [0.01, 0.5]}},
}
)
manager = HyperoptManager(config)
assert manager.algorithm == hyperopt.rand.suggest
config = V1Hyperopt.from_dict(
{
"concurrency": 2,
"algorithm": "anneal",
"numRuns": 1,
"params": {"param": {"kind": "uniform", "value": [0.01, 0.5]}},
}
)
manager = HyperoptManager(config)
assert manager.algorithm == hyperopt.anneal.suggest
config = V1Hyperopt.from_dict(
{
"concurrency": 2,
"algorithm": "tpe",
"numRuns": 1,
"params": {"param": {"kind": "uniform", "value": [0.01, 0.5]}},
}
)
manager = HyperoptManager(config)
assert manager.algorithm == hyperopt.tpe.suggest
def test_get_tpe_suggestions(self):
config = V1Hyperopt.from_dict({
"concurrency": 2,
"algorithm": "tpe",
"numRuns": 1,
"params": {
"lr": {
"kind": "uniform",
"value": [0.01, 0.5]
},
"dropout": {
"kind": "uniform",
"value": [0.01, 0.99]
},
"batch": {
"kind": "choice",
"value": [32, 64, 126, 256]
},
"optimizer": {
"kind": "choice",
"value": ["sgd", "adagrad", "adam", "ftrl"],
},
},
})
suggestion = HyperoptManager(config).get_suggestions()[0]
self.assertTrue(0.99 >= suggestion["dropout"] >= 0.01)
self.assertTrue(0.5 >= suggestion["lr"] >= 0.01)
self.assertTrue(suggestion["batch"] in [32, 64, 126, 256])
self.assertTrue(
suggestion["optimizer"] in ["sgd", "adagrad", "adam", "ftrl"])
config = V1Hyperopt.from_dict({
"concurrency": 2,
"algorithm": "tpe",
"numRuns": 10,
"params": {
"lr": {
"kind": "uniform",
"value": [0.01, 0.5]
},
"dropout": {
"kind": "uniform",
"value": [0.01, 0.99]
},
"batch": {
"kind": "choice",
"value": [32, 64, 126, 256]
},
"optimizer": {
"kind": "choice",
"value": ["sgd", "adagrad", "adam", "ftrl"],
},
},
})
assert len(HyperoptManager(config).get_suggestions()) == 10
def hyperopt(matrix, configs, metrics):
"""Create suggestions based on hyperopt."""
from polyaxon.polyflow import V1Hyperopt
from polyaxon.polytune.search_managers.hyperopt.manager import HyperoptManager
matrix = V1Hyperopt.read(matrix)
suggestions = HyperoptManager(config=matrix).get_suggestions(
configs=configs, metrics=metrics)
log_suggestions(suggestions)
Printer.print_success("Suggestions generated with hyperopt")
def test_search_space(self):
config = V1Hyperopt.from_dict({
"concurrency": 2,
"algorithm": "rand",
"numRuns": 1,
"params": {
"param1": {
"kind": "uniform",
"value": [0.01, 0.5]
},
"param2": {
"kind": "quniform",
"value": [0.01, 0.99, 0.1]
},
"param3": {
"kind": "normal",
"value": [0, 0.99]
},
"param4": {
"kind": "choice",
"value": [32, 64, 126, 256]
},
"param5": {
"kind": "choice",
"value": ["sgd", "adagrad", "adam", "ftrl"],
},
"param6": {
"kind": "linspace",
"value": [0, 10, 1]
},
"param7": {
"kind": "geomspace",
"value": [0.1, 1, 1]
},
},
})
manager = HyperoptManager(config)
assert set(manager._param_to_value.keys()) == {
"param4",
"param5",
"param6",
"param7",
}
assert set(manager._search_space.keys()) == {
"param1",
"param2",
"param3",
"param4",
"param5",
"param6",
"param7",
}
def hyperopt(matrix, join, iteration):
"""Create suggestions based on hyperopt."""
from polyaxon.client import RunClient
from polyaxon.polyflow import V1Hyperopt, V1Join
from polyaxon.polytune.iteration_lineage import (
get_iteration_definition,
handle_iteration,
handle_iteration_failure,
)
from polyaxon.polytune.search_managers.hyperopt.manager import HyperoptManager
matrix = V1Hyperopt.read(matrix)
join = V1Join.read(join)
client = RunClient()
values = get_iteration_definition(
client=client,
iteration=iteration,
join=join,
optimization_metric=matrix.metric.name,
)
if not values:
return
run_uuids, configs, metrics = values
retry = 1
exp = None
suggestions = None
while retry < 3:
try:
suggestions = HyperoptManager(config=matrix).get_suggestions(
configs=configs, metrics=metrics)
exp = None
break
except Exception as exp:
retry += 1
logger.warning(exp)
if exp:
handle_iteration_failure(client=client, exp=exp)
return
handle_iteration(
client=client,
iteration=iteration,
suggestions=suggestions,
)
def hyperopt(matrix, configs, metrics, iteration):
"""Create suggestions based on hyperopt."""
from polyaxon.client import RunClient
from polyaxon.polyflow import V1Hyperopt
from polyaxon.polytune.iteration_lineage import (
handle_iteration,
handle_iteration_failure,
)
from polyaxon.polytune.search_managers.hyperopt.manager import HyperoptManager
matrix = V1Hyperopt.read(matrix)
if configs:
configs = ujson.loads(configs)
if metrics:
metrics = ujson.loads(metrics)
client = RunClient()
retry = 1
exp = None
suggestions = None
while retry < 3:
try:
suggestions = HyperoptManager(config=matrix).get_suggestions(
configs=configs, metrics=metrics)
exp = None
break
except Exception as e:
retry += 1
logger.warning(e)
exp = e
if exp:
handle_iteration_failure(client=client, exp=exp)
return
handle_iteration(
client=client,
suggestions=suggestions,
)
