def test_getGridsearchSolver(self):
config = {
"hyperparameter": {
"value 1": {
"domain": "uniform",
"data": [0, 20],
"type": int,
"frequency": 11
},
"value 2": {
"domain": "normal",
"data": [0, 20.0],
"type": float,
"frequency": 11
},
"value 3": {
"domain": "loguniform",
"data": [1, 10000],
"type": float,
"frequency": 11
},
"categorical": {
"domain": "categorical",
"data": ["a", "b"],
"type": str,
"frequency": 1
}
}
}
res_labels = ['value 1', 'value 2', 'value 3', 'categorical']
res_values = [[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20],
[
0.0, 5.467452952462635, 8.663855974622837,
9.755510546899107, 9.973002039367397, 10.0,
10.026997960632603, 10.244489453100893,
11.336144025377163, 14.532547047537365, 20.0
],
[
1.0, 2.51188643150958, 6.309573444801933,
15.848931924611136, 39.810717055349734,
100.00000000000004, 251.18864315095806,
630.9573444801938, 1584.8931924611143,
3981.071705534977, 10000.00000000001
], ['a', 'b']]
project = HyppopyProject(config)
solver = SolverPool.get("gridsearch", project)
self.assertTrue(isinstance(solver, GridsearchSolver))
searchspace = solver.convert_searchspace(config["hyperparameter"])
for n in range(len(res_labels)):
self.assertEqual(res_labels[n], searchspace[0][n])
for i in range(3):
self.assertAlmostEqual(res_values[i], searchspace[1][i])
self.assertEqual(res_values[3], searchspace[1][3])
def project(self, value):
"""
Set HyppopyProject instance
:param value: [HyppopyProject] project instance
"""
if isinstance(value, dict):
self._project = HyppopyProject(value)
elif isinstance(value, HyppopyProject):
self._project = value
else:
msg = "Input error, project_manager of type: {} not allowed!".format(
type(value))
LOG.error(msg)
raise TypeError(msg)
self._check_project()
def test_categorical(self):
config = {
"hyperparameter": {
"C": {
"domain": "uniform",
"data": [1, 20],
"type": int
},
"gamma": {
"domain": "loguniform",
"data": [0.0, 20.0],
"type": float
},
"kernel": {
"domain": "categorical",
"data": ["linear", "sigmoid", "poly", "rbf"],
"type": str
},
"with_ovr": {
"domain": "categorical",
"data": [True, False],
"type": bool
}
},
"max_iterations": 300
}
project = HyppopyProject(config=config)
from sklearn.svm import SVC
from sklearn.datasets import load_iris
from sklearn.model_selection import cross_val_score
iris_data = load_iris()
data = [iris_data.data, iris_data.target]
def my_loss_function(data, params):
if params["with_ovr"]:
params["decision_function_shape"] = "ovr"
else:
params["decision_function_shape"] = "ovo"
del params["with_ovr"]
clf = SVC(**params)
return -cross_val_score(estimator=clf, X=data[0], y=data[1],
cv=3).mean()
blackbox = BlackboxFunction(blackbox_func=my_loss_function, data=data)
solver = HyperoptSolver(project=project)
solver.blackbox = blackbox
solver.run()
def test_solver_complete(self):
config = {
"hyperparameter": {
"axis_00": {
"domain": "normal",
"data": [300, 800],
"type": "float"
},
"axis_01": {
"domain": "normal",
"data": [-1, 1],
"type": "float"
},
"axis_02": {
"domain": "uniform",
"data": [0, 10],
"type": "float"
}
},
"settings": {
"solver": {
"max_iterations": 100
},
"custom": {}
}
}
project = HyppopyProject(config)
solver = OptunaSolver(project)
vfunc = VirtualFunction()
vfunc.load_default()
solver.blackbox = vfunc
solver.run(print_stats=False)
df, best = solver.get_results()
self.assertTrue(300 <= best['axis_00'] <= 800)
self.assertTrue(-1 <= best['axis_01'] <= 1)
self.assertTrue(0 <= best['axis_02'] <= 10)
for status in df['status']:
self.assertTrue(status)
for loss in df['losses']:
self.assertTrue(isinstance(loss, float))
def __init__(self, project=None):
config = {
"hyperparameter": {
"gamma": {
"domain": "uniform",
"data": [0.0001, 20.0],
"type": float
},
"kernel": {
"domain": "categorical",
"data": ["linear", "sigmoid", "poly", "rbf"],
"type": str
}
},
"foo1": 300,
"goo": 1.0
}
project = HyppopyProject(config)
HyppopySolver.__init__(self, project)
self._searchspace = None
def test_solver_loguniform(self):
config = {
"hyperparameter": {
"axis_00": {
"domain": "loguniform",
"data": [0.00001, 300],
"type": float,
"frequency": 21
},
"axis_01": {
"domain": "loguniform",
"data": [0.00001, 0.8],
"type": float,
"frequency": 21
},
"axis_02": {
"domain": "loguniform",
"data": [4, 6],
"type": float,
"frequency": 21
}
}
}
project = HyppopyProject(config)
solver = GridsearchSolver(project)
vfunc = FunctionSimulator()
vfunc.load_default()
solver.blackbox = vfunc
solver.run(print_stats=False)
df, best = solver.get_results()
self.assertAlmostEqual(best['axis_00'], 299.999, places=1)
self.assertAlmostEqual(best['axis_01'], 0.25869, places=1)
self.assertAlmostEqual(best['axis_02'], 5.10169, places=1)
for status in df['status']:
self.assertTrue(status)
for loss in df['losses']:
self.assertTrue(isinstance(loss, float))
def test_getOptunitySolver(self):
config = {
"hyperparameter": {
"axis_00": {
"domain": "uniform",
"data": [300, 800],
"type": float
},
"axis_01": {
"domain": "uniform",
"data": [-1, 1],
"type": float
},
"axis_02": {
"domain": "uniform",
"data": [0, 10],
"type": float
}
},
"max_iterations": 100
}
project = HyppopyProject(config)
solver = SolverPool.get("optunity", project)
self.assertTrue(isinstance(solver, OptunitySolver))
vfunc = FunctionSimulator()
vfunc.load_default()
solver.blackbox = vfunc
solver.run(print_stats=False)
df, best = solver.get_results()
self.assertTrue(300 <= best['axis_00'] <= 800)
self.assertTrue(-1 <= best['axis_01'] <= 1)
self.assertTrue(0 <= best['axis_02'] <= 10)
for status in df['status']:
self.assertTrue(status)
for loss in df['losses']:
self.assertTrue(isinstance(loss, float))
def test_solver_loguniform(self):
config = {
"hyperparameter": {
"axis_00": {
"domain": "normal",
"data": [500, 650],
"type": float
},
"axis_01": {
"domain": "loguniform",
"data": [0.001, 1],
"type": float
},
"axis_02": {
"domain": "normal",
"data": [4.5, 5.5],
"type": float
}
},
"max_iterations": 500,
}
project = HyppopyProject(config)
solver = HyperoptSolver(project)
vfunc = FunctionSimulator()
vfunc.load_default()
solver.blackbox = vfunc
solver.run(print_stats=False)
df, best = solver.get_results()
self.assertTrue(575 <= best['axis_00'] <= 585)
self.assertTrue(0.001 <= best['axis_01'] <= 1.0)
self.assertTrue(4.7 <= best['axis_02'] <= 5.3)
for status in df['status']:
self.assertTrue(status)
for loss in df['losses']:
self.assertTrue(isinstance(loss, float))
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
import matplotlib.pylab as plt
from hyppopy.SolverPool import SolverPool
from hyppopy.HyppopyProject import HyppopyProject
from hyppopy.VirtualFunction import VirtualFunction
from hyppopy.BlackboxFunction import BlackboxFunction
project = HyppopyProject()
project.add_hyperparameter(name="axis_00",
domain="uniform",
data=[0, 1],
dtype="float")
project.add_hyperparameter(name="axis_01",
domain="uniform",
data=[0, 1],
dtype="float")
project.add_hyperparameter(name="axis_02",
domain="uniform",
data=[0, 1],
dtype="float")
project.add_hyperparameter(name="axis_03",
domain="uniform",
data=[0, 1],
def __init__(self, project=None):
project = HyppopyProject({})
HyppopySolver.__init__(self, project)
self._searchspace = None
# All rights reserved.
#
# This software is distributed WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.
#
# See LICENSE
import matplotlib.pylab as plt
from hyppopy.SolverPool import SolverPool
from hyppopy.HyppopyProject import HyppopyProject
from hyppopy.FunctionSimulator import FunctionSimulator
from hyppopy.BlackboxFunction import BlackboxFunction
project = HyppopyProject()
project.add_hyperparameter(name="axis_00", domain="uniform", data=[0, 1], type=float)
project.add_hyperparameter(name="axis_01", domain="uniform", data=[0, 1], type=float)
project.add_hyperparameter(name="axis_02", domain="uniform", data=[0, 1], type=float)
project.add_hyperparameter(name="axis_03", domain="uniform", data=[0, 1], type=float)
project.add_hyperparameter(name="axis_04", domain="uniform", data=[0, 1], type=float)
project.add_setting("max_iterations", 500)
project.add_setting("solver", "randomsearch")
plt.ion()
fig, axes = plt.subplots(nrows=2, ncols=3, figsize=(12, 8), sharey=True)
plot_data = {"iterations": [],
"loss": [],
"axis_00": [],
"axis_01": [],
"axis_02": [],
"domain": "uniform",
"data": [0.0001, 20.0],
"type": float
},
"kernel": {
"domain": "categorical",
"data": ["linear", "sigmoid", "poly", "rbf"],
"type": str
}
},
"max_iterations": 500
}
# When creating a HyppopyProject instance we
# pass the config dictionary to the constructor.
project = HyppopyProject(config=config)
# When building the project programmatically we can also use the methods
# add_hyperparameter and add_settings
project = HyppopyProject()
project.add_hyperparameter(name="C",
domain="uniform",
data=[0.0001, 20],
dtype="float")
project.add_hyperparameter(name="kernel",
domain="categorical",
data=["linear", "sigmoid"],
dtype="str")
project.set_settings(max_iterations=500)
# The custom section can be used freely
def test_project_creation(self):
config = {
"hyperparameter": {
"C": {
"domain": "uniform",
"data": [0.0001, 20],
"type": "float"
},
"kernel": {
"domain": "categorical",
"data": ["linear", "sigmoid", "poly", "rbf"],
"type": "str"
}
},
"settings": {
"solver": {
"max_iterations": 300
},
"custom": {
"param1": 1,
"param2": 2,
"function": foo
}
}}
project = HyppopyProject()
project.set_config(config)
self.assertEqual(project.hyperparameter["C"]["domain"], "uniform")
self.assertEqual(project.hyperparameter["C"]["data"], [0.0001, 20])
self.assertEqual(project.hyperparameter["C"]["type"], "float")
self.assertEqual(project.hyperparameter["kernel"]["domain"], "categorical")
self.assertEqual(project.hyperparameter["kernel"]["data"], ["linear", "sigmoid", "poly", "rbf"])
self.assertEqual(project.hyperparameter["kernel"]["type"], "str")
self.assertEqual(project.solver_max_iterations, 300)
self.assertEqual(project.custom_param1, 1)
self.assertEqual(project.custom_param2, 2)
self.assertEqual(project.custom_function(2, 3), 5)
self.assertTrue(project.get_typeof("C") is float)
self.assertTrue(project.get_typeof("kernel") is str)
project.clear()
self.assertTrue(len(project.hyperparameter) == 0)
self.assertTrue(len(project.settings) == 0)
self.assertTrue("solver_max_iterations" not in project.__dict__.keys())
self.assertTrue("custom_param1" not in project.__dict__.keys())
self.assertTrue("custom_param2" not in project.__dict__.keys())
self.assertTrue("custom_function" not in project.__dict__.keys())
project.add_hyperparameter(name="C", domain="uniform", data=[0.0001, 20], dtype="float")
project.add_hyperparameter(name="kernel", domain="categorical", data=["linear", "sigmoid", "poly", "rbf"], dtype="str")
self.assertEqual(project.hyperparameter["C"]["domain"], "uniform")
self.assertEqual(project.hyperparameter["C"]["data"], [0.0001, 20])
self.assertEqual(project.hyperparameter["C"]["type"], "float")
self.assertEqual(project.hyperparameter["kernel"]["domain"], "categorical")
self.assertEqual(project.hyperparameter["kernel"]["data"], ["linear", "sigmoid", "poly", "rbf"])
self.assertEqual(project.hyperparameter["kernel"]["type"], "str")
project.add_settings("solver", "max_iterations", 500)
self.assertEqual(project.solver_max_iterations, 500)
project.add_settings("solver", "max_iterations", 200)
self.assertEqual(project.solver_max_iterations, 200)
"domain": "uniform",
"data": [-10.0, 10.0],
"type": "float"
}
},
"settings": {
"solver": {
"max_iterations": 500
},
"custom": {}
}}
# When creating a HyppopyProject instance we
# pass the config dictionary to the constructor.
project = HyppopyProject(config=config)
# The user defined loss function
def my_loss_function(x, y):
return x**2+y**2
# create a solver instance
solver = HyperoptSolver(project)
# pass the loss function to the solver
solver.blackbox = my_loss_function
# run the solver
solver.run()
df, best = solver.get_results()
def test_project_creation(self):
config = {
"hyperparameter": {
"C": {
"domain": "uniform",
"data": [0.0001, 20],
"type": float
},
"kernel": {
"domain": "categorical",
"data": ["linear", "sigmoid", "poly", "rbf"],
"type": str
}
},
"max_iterations": 300,
"param1": 1,
"param2": 2,
"function": foo
}
project = HyppopyProject()
project.set_config(config)
self.assertEqual(project.hyperparameter["C"]["domain"], "uniform")
self.assertEqual(project.hyperparameter["C"]["data"], [0.0001, 20])
self.assertTrue(project.hyperparameter["C"]["type"] is float)
self.assertEqual(project.hyperparameter["kernel"]["domain"],
"categorical")
self.assertEqual(project.hyperparameter["kernel"]["data"],
["linear", "sigmoid", "poly", "rbf"])
self.assertTrue(project.hyperparameter["kernel"]["type"] is str)
self.assertEqual(project.max_iterations, 300)
self.assertEqual(project.param1, 1)
self.assertEqual(project.param2, 2)
self.assertEqual(project.function(2, 3), 5)
self.assertTrue(project.get_typeof("C") is float)
self.assertTrue(project.get_typeof("kernel") is str)
project = HyppopyProject()
project.add_hyperparameter(name="C",
domain="uniform",
data=[0.0001, 20],
type=float)
project.add_hyperparameter(name="kernel",
domain="categorical",
data=["linear", "sigmoid", "poly", "rbf"],
type=str)
self.assertEqual(project.hyperparameter["C"]["domain"], "uniform")
self.assertEqual(project.hyperparameter["C"]["data"], [0.0001, 20])
self.assertTrue(project.hyperparameter["C"]["type"] is float)
self.assertEqual(project.hyperparameter["kernel"]["domain"],
"categorical")
self.assertEqual(project.hyperparameter["kernel"]["data"],
["linear", "sigmoid", "poly", "rbf"])
self.assertTrue(project.hyperparameter["kernel"]["type"] is str)
project.set_settings(max_iterations=500)
self.assertEqual(project.max_iterations, 500)
project.add_setting("my_param", 42)
self.assertEqual(project.my_param, 42)
project.add_setting("max_iterations", 200)
self.assertEqual(project.max_iterations, 200)
def compute_deviation(solver_name, vfunc_id, iterations, N, fname):
project = HyppopyProject()
project.add_hyperparameter(name="axis_00",
domain="uniform",
data=[0, 1],
type=float)
project.add_hyperparameter(name="axis_01",
domain="uniform",
data=[0, 1],
type=float)
project.add_hyperparameter(name="axis_02",
domain="uniform",
data=[0, 1],
type=float)
project.add_hyperparameter(name="axis_03",
domain="uniform",
data=[0, 1],
type=float)
project.add_hyperparameter(name="axis_04",
domain="uniform",
data=[0, 1],
type=float)
vfunc = FunctionSimulator()
vfunc.load_default(vfunc_id)
minima = vfunc.minima()
def my_loss_function(data, params):
return vfunc(**params)
blackbox = BlackboxFunction(data=[], blackbox_func=my_loss_function)
results = {}
results["gt"] = []
for mini in minima:
results["gt"].append(np.median(mini[0]))
for iter in iterations:
results[iter] = {
"minima": {},
"distance": {},
"duration": None,
"set_difference": None,
"loss": None,
"loss_history": {}
}
for i in range(vfunc.dims()):
results[iter]["minima"]["axis_0{}".format(i)] = []
results[iter]["distance"]["axis_0{}".format(i)] = []
project.add_setting("max_iterations", iter)
project.add_setting("solver", solver_name)
solver = SolverPool.get(project=project)
solver.blackbox = blackbox
axis_minima = []
best_losses = []
best_sets_diff = []
for i in range(vfunc.dims()):
axis_minima.append([])
loss_history = []
durations = []
for n in range(N):
print("\rSolver={} iteration={} round={}".format(solver, iter, n),
end="")
start = time.time()
solver.run(print_stats=False)
end = time.time()
durations.append(end - start)
df, best = solver.get_results()
loss_history.append(np.flip(np.sort(df['losses'].values)))
best_row = df['losses'].idxmin()
best_losses.append(df['losses'][best_row])
best_sets_diff.append(
abs(df['axis_00'][best_row] - best['axis_00']) +
abs(df['axis_01'][best_row] - best['axis_01']) +
abs(df['axis_02'][best_row] - best['axis_02']) +
abs(df['axis_03'][best_row] - best['axis_03']) +
abs(df['axis_04'][best_row] - best['axis_04']))
for i in range(vfunc.dims()):
tmp = df['axis_0{}'.format(i)][best_row]
axis_minima[i].append(tmp)
results[iter]["loss_history"] = loss_history
for i in range(vfunc.dims()):
results[iter]["minima"]["axis_0{}".format(i)] = [
np.mean(axis_minima[i]),
np.std(axis_minima[i])
]
dist = np.sqrt((axis_minima[i] - results["gt"][i])**2)
results[iter]["distance"]["axis_0{}".format(i)] = [
np.mean(dist), np.std(dist)
]
results[iter]["loss"] = [np.mean(best_losses), np.std(best_losses)]
results[iter]["set_difference"] = sum(best_sets_diff)
results[iter]["duration"] = np.mean(durations)
file = open(fname, 'wb')
pickle.dump(results, file)
file.close()
def new_project(self, name="HyppopyProject", config=None):
if name in self._projects.keys():
name = self.check_projectname(name)
self._projects[name] = HyppopyProject(config)
self._current_project = self._projects[name]
return self._current_project
def compute_deviation(solver_name, vfunc_id, iterations, N, fname):
project = HyppopyProject()
project.add_hyperparameter(name="axis_00", domain="uniform", data=[0, 1], dtype="float")
project.add_hyperparameter(name="axis_01", domain="uniform", data=[0, 1], dtype="float")
project.add_hyperparameter(name="axis_02", domain="uniform", data=[0, 1], dtype="float")
project.add_hyperparameter(name="axis_03", domain="uniform", data=[0, 1], dtype="float")
project.add_hyperparameter(name="axis_04", domain="uniform", data=[0, 1], dtype="float")
vfunc = VirtualFunction()
vfunc.load_default(vfunc_id)
minima = vfunc.minima()
def my_loss_function(data, params):
return vfunc(**params)
blackbox = BlackboxFunction(data=[], blackbox_func=my_loss_function)
results = {}
results["gt"] = []
for mini in minima:
results["gt"].append(np.median(mini[0]))
for iter in iterations:
results[iter] = {"minima": {}, "loss": None}
for i in range(vfunc.dims()):
results[iter]["minima"]["axis_0{}".format(i)] = []
project.add_settings(section="solver", name="max_iterations", value=iter)
project.add_settings(section="custom", name="use_solver", value=solver_name)
solver = SolverPool.get(project=project)
solver.blackbox = blackbox
axis_minima = []
best_losses = []
for i in range(vfunc.dims()):
axis_minima.append([])
for n in range(N):
print("\rSolver={} iteration={} round={}".format(solver, iter, n), end="")
solver.run(print_stats=False)
df, best = solver.get_results()
best_row = df['losses'].idxmin()
best_losses.append(df['losses'][best_row])
for i in range(vfunc.dims()):
tmp = df['axis_0{}'.format(i)][best_row]
axis_minima[i].append(tmp)
for i in range(vfunc.dims()):
results[iter]["minima"]["axis_0{}".format(i)] = [np.mean(axis_minima[i]), np.std(axis_minima[i])]
results[iter]["loss"] = [np.mean(best_losses), np.std(best_losses)]
file = open(fname, 'wb')
pickle.dump(results, file)
file.close()
"kernel": {
"domain": "categorical",
"data": ["linear", "sigmoid", "poly", "rbf"],
"type": "str"
}
},
"settings": {
"solver": {
"max_iterations": 500
},
"custom": {}
}}
# When creating a HyppopyProject instance we
# pass the config dictionary to the constructor.
project = HyppopyProject(config=config)
# When building the project programmatically we can also use the methods
# add_hyperparameter and add_settings
project.clear()
project.add_hyperparameter(name="C", domain="uniform", data=[0.0001, 20], dtype="float")
project.add_hyperparameter(name="kernel", domain="categorical", data=["linear", "sigmoid"], dtype="str")
project.add_settings(section="solver", name="max_iterations", value=500)
# The custom section can be used freely
project.add_settings(section="custom", name="my_var", value=10)
# Settings are automatically transformed to member variables of the project class with the section as prefix
if project.solver_max_iterations < 1000 and project.custom_my_var == 10:
print("Project configured!")
