def test_getGridsearchSolver(self): config = { "hyperparameter": { "value 1": { "domain": "uniform", "data": [0, 20], "type": int, "frequency": 11 }, "value 2": { "domain": "normal", "data": [0, 20.0], "type": float, "frequency": 11 }, "value 3": { "domain": "loguniform", "data": [1, 10000], "type": float, "frequency": 11 }, "categorical": { "domain": "categorical", "data": ["a", "b"], "type": str, "frequency": 1 } } } res_labels = ['value 1', 'value 2', 'value 3', 'categorical'] res_values = [[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20], [ 0.0, 5.467452952462635, 8.663855974622837, 9.755510546899107, 9.973002039367397, 10.0, 10.026997960632603, 10.244489453100893, 11.336144025377163, 14.532547047537365, 20.0 ], [ 1.0, 2.51188643150958, 6.309573444801933, 15.848931924611136, 39.810717055349734, 100.00000000000004, 251.18864315095806, 630.9573444801938, 1584.8931924611143, 3981.071705534977, 10000.00000000001 ], ['a', 'b']] project = HyppopyProject(config) solver = SolverPool.get("gridsearch", project) self.assertTrue(isinstance(solver, GridsearchSolver)) searchspace = solver.convert_searchspace(config["hyperparameter"]) for n in range(len(res_labels)): self.assertEqual(res_labels[n], searchspace[0][n]) for i in range(3): self.assertAlmostEqual(res_values[i], searchspace[1][i]) self.assertEqual(res_values[3], searchspace[1][3])
def project(self, value): """ Set HyppopyProject instance :param value: [HyppopyProject] project instance """ if isinstance(value, dict): self._project = HyppopyProject(value) elif isinstance(value, HyppopyProject): self._project = value else: msg = "Input error, project_manager of type: {} not allowed!".format( type(value)) LOG.error(msg) raise TypeError(msg) self._check_project()
def test_categorical(self): config = { "hyperparameter": { "C": { "domain": "uniform", "data": [1, 20], "type": int }, "gamma": { "domain": "loguniform", "data": [0.0, 20.0], "type": float }, "kernel": { "domain": "categorical", "data": ["linear", "sigmoid", "poly", "rbf"], "type": str }, "with_ovr": { "domain": "categorical", "data": [True, False], "type": bool } }, "max_iterations": 300 } project = HyppopyProject(config=config) from sklearn.svm import SVC from sklearn.datasets import load_iris from sklearn.model_selection import cross_val_score iris_data = load_iris() data = [iris_data.data, iris_data.target] def my_loss_function(data, params): if params["with_ovr"]: params["decision_function_shape"] = "ovr" else: params["decision_function_shape"] = "ovo" del params["with_ovr"] clf = SVC(**params) return -cross_val_score(estimator=clf, X=data[0], y=data[1], cv=3).mean() blackbox = BlackboxFunction(blackbox_func=my_loss_function, data=data) solver = HyperoptSolver(project=project) solver.blackbox = blackbox solver.run()
def test_solver_complete(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", "data": [300, 800], "type": "float" }, "axis_01": { "domain": "normal", "data": [-1, 1], "type": "float" }, "axis_02": { "domain": "uniform", "data": [0, 10], "type": "float" } }, "settings": { "solver": { "max_iterations": 100 }, "custom": {} } } project = HyppopyProject(config) solver = OptunaSolver(project) vfunc = VirtualFunction() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertTrue(300 <= best['axis_00'] <= 800) self.assertTrue(-1 <= best['axis_01'] <= 1) self.assertTrue(0 <= best['axis_02'] <= 10) for status in df['status']: self.assertTrue(status) for loss in df['losses']: self.assertTrue(isinstance(loss, float))
def __init__(self, project=None): config = { "hyperparameter": { "gamma": { "domain": "uniform", "data": [0.0001, 20.0], "type": float }, "kernel": { "domain": "categorical", "data": ["linear", "sigmoid", "poly", "rbf"], "type": str } }, "foo1": 300, "goo": 1.0 } project = HyppopyProject(config) HyppopySolver.__init__(self, project) self._searchspace = None
def test_solver_loguniform(self): config = { "hyperparameter": { "axis_00": { "domain": "loguniform", "data": [0.00001, 300], "type": float, "frequency": 21 }, "axis_01": { "domain": "loguniform", "data": [0.00001, 0.8], "type": float, "frequency": 21 }, "axis_02": { "domain": "loguniform", "data": [4, 6], "type": float, "frequency": 21 } } } project = HyppopyProject(config) solver = GridsearchSolver(project) vfunc = FunctionSimulator() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertAlmostEqual(best['axis_00'], 299.999, places=1) self.assertAlmostEqual(best['axis_01'], 0.25869, places=1) self.assertAlmostEqual(best['axis_02'], 5.10169, places=1) for status in df['status']: self.assertTrue(status) for loss in df['losses']: self.assertTrue(isinstance(loss, float))
def test_getOptunitySolver(self): config = { "hyperparameter": { "axis_00": { "domain": "uniform", "data": [300, 800], "type": float }, "axis_01": { "domain": "uniform", "data": [-1, 1], "type": float }, "axis_02": { "domain": "uniform", "data": [0, 10], "type": float } }, "max_iterations": 100 } project = HyppopyProject(config) solver = SolverPool.get("optunity", project) self.assertTrue(isinstance(solver, OptunitySolver)) vfunc = FunctionSimulator() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertTrue(300 <= best['axis_00'] <= 800) self.assertTrue(-1 <= best['axis_01'] <= 1) self.assertTrue(0 <= best['axis_02'] <= 10) for status in df['status']: self.assertTrue(status) for loss in df['losses']: self.assertTrue(isinstance(loss, float))
def test_solver_loguniform(self): config = { "hyperparameter": { "axis_00": { "domain": "normal", "data": [500, 650], "type": float }, "axis_01": { "domain": "loguniform", "data": [0.001, 1], "type": float }, "axis_02": { "domain": "normal", "data": [4.5, 5.5], "type": float } }, "max_iterations": 500, } project = HyppopyProject(config) solver = HyperoptSolver(project) vfunc = FunctionSimulator() vfunc.load_default() solver.blackbox = vfunc solver.run(print_stats=False) df, best = solver.get_results() self.assertTrue(575 <= best['axis_00'] <= 585) self.assertTrue(0.001 <= best['axis_01'] <= 1.0) self.assertTrue(4.7 <= best['axis_02'] <= 5.3) for status in df['status']: self.assertTrue(status) for loss in df['losses']: self.assertTrue(isinstance(loss, float))
# All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE import matplotlib.pylab as plt from hyppopy.SolverPool import SolverPool from hyppopy.HyppopyProject import HyppopyProject from hyppopy.VirtualFunction import VirtualFunction from hyppopy.BlackboxFunction import BlackboxFunction project = HyppopyProject() project.add_hyperparameter(name="axis_00", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_01", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_02", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_03", domain="uniform", data=[0, 1],
def __init__(self, project=None): project = HyppopyProject({}) HyppopySolver.__init__(self, project) self._searchspace = None
# All rights reserved. # # This software is distributed WITHOUT ANY WARRANTY; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. # # See LICENSE import matplotlib.pylab as plt from hyppopy.SolverPool import SolverPool from hyppopy.HyppopyProject import HyppopyProject from hyppopy.FunctionSimulator import FunctionSimulator from hyppopy.BlackboxFunction import BlackboxFunction project = HyppopyProject() project.add_hyperparameter(name="axis_00", domain="uniform", data=[0, 1], type=float) project.add_hyperparameter(name="axis_01", domain="uniform", data=[0, 1], type=float) project.add_hyperparameter(name="axis_02", domain="uniform", data=[0, 1], type=float) project.add_hyperparameter(name="axis_03", domain="uniform", data=[0, 1], type=float) project.add_hyperparameter(name="axis_04", domain="uniform", data=[0, 1], type=float) project.add_setting("max_iterations", 500) project.add_setting("solver", "randomsearch") plt.ion() fig, axes = plt.subplots(nrows=2, ncols=3, figsize=(12, 8), sharey=True) plot_data = {"iterations": [], "loss": [], "axis_00": [], "axis_01": [], "axis_02": [],
"domain": "uniform", "data": [0.0001, 20.0], "type": float }, "kernel": { "domain": "categorical", "data": ["linear", "sigmoid", "poly", "rbf"], "type": str } }, "max_iterations": 500 } # When creating a HyppopyProject instance we # pass the config dictionary to the constructor. project = HyppopyProject(config=config) # When building the project programmatically we can also use the methods # add_hyperparameter and add_settings project = HyppopyProject() project.add_hyperparameter(name="C", domain="uniform", data=[0.0001, 20], dtype="float") project.add_hyperparameter(name="kernel", domain="categorical", data=["linear", "sigmoid"], dtype="str") project.set_settings(max_iterations=500) # The custom section can be used freely
def test_project_creation(self): config = { "hyperparameter": { "C": { "domain": "uniform", "data": [0.0001, 20], "type": "float" }, "kernel": { "domain": "categorical", "data": ["linear", "sigmoid", "poly", "rbf"], "type": "str" } }, "settings": { "solver": { "max_iterations": 300 }, "custom": { "param1": 1, "param2": 2, "function": foo } }} project = HyppopyProject() project.set_config(config) self.assertEqual(project.hyperparameter["C"]["domain"], "uniform") self.assertEqual(project.hyperparameter["C"]["data"], [0.0001, 20]) self.assertEqual(project.hyperparameter["C"]["type"], "float") self.assertEqual(project.hyperparameter["kernel"]["domain"], "categorical") self.assertEqual(project.hyperparameter["kernel"]["data"], ["linear", "sigmoid", "poly", "rbf"]) self.assertEqual(project.hyperparameter["kernel"]["type"], "str") self.assertEqual(project.solver_max_iterations, 300) self.assertEqual(project.custom_param1, 1) self.assertEqual(project.custom_param2, 2) self.assertEqual(project.custom_function(2, 3), 5) self.assertTrue(project.get_typeof("C") is float) self.assertTrue(project.get_typeof("kernel") is str) project.clear() self.assertTrue(len(project.hyperparameter) == 0) self.assertTrue(len(project.settings) == 0) self.assertTrue("solver_max_iterations" not in project.__dict__.keys()) self.assertTrue("custom_param1" not in project.__dict__.keys()) self.assertTrue("custom_param2" not in project.__dict__.keys()) self.assertTrue("custom_function" not in project.__dict__.keys()) project.add_hyperparameter(name="C", domain="uniform", data=[0.0001, 20], dtype="float") project.add_hyperparameter(name="kernel", domain="categorical", data=["linear", "sigmoid", "poly", "rbf"], dtype="str") self.assertEqual(project.hyperparameter["C"]["domain"], "uniform") self.assertEqual(project.hyperparameter["C"]["data"], [0.0001, 20]) self.assertEqual(project.hyperparameter["C"]["type"], "float") self.assertEqual(project.hyperparameter["kernel"]["domain"], "categorical") self.assertEqual(project.hyperparameter["kernel"]["data"], ["linear", "sigmoid", "poly", "rbf"]) self.assertEqual(project.hyperparameter["kernel"]["type"], "str") project.add_settings("solver", "max_iterations", 500) self.assertEqual(project.solver_max_iterations, 500) project.add_settings("solver", "max_iterations", 200) self.assertEqual(project.solver_max_iterations, 200)
"domain": "uniform", "data": [-10.0, 10.0], "type": "float" } }, "settings": { "solver": { "max_iterations": 500 }, "custom": {} }} # When creating a HyppopyProject instance we # pass the config dictionary to the constructor. project = HyppopyProject(config=config) # The user defined loss function def my_loss_function(x, y): return x**2+y**2 # create a solver instance solver = HyperoptSolver(project) # pass the loss function to the solver solver.blackbox = my_loss_function # run the solver solver.run() df, best = solver.get_results()
def test_project_creation(self): config = { "hyperparameter": { "C": { "domain": "uniform", "data": [0.0001, 20], "type": float }, "kernel": { "domain": "categorical", "data": ["linear", "sigmoid", "poly", "rbf"], "type": str } }, "max_iterations": 300, "param1": 1, "param2": 2, "function": foo } project = HyppopyProject() project.set_config(config) self.assertEqual(project.hyperparameter["C"]["domain"], "uniform") self.assertEqual(project.hyperparameter["C"]["data"], [0.0001, 20]) self.assertTrue(project.hyperparameter["C"]["type"] is float) self.assertEqual(project.hyperparameter["kernel"]["domain"], "categorical") self.assertEqual(project.hyperparameter["kernel"]["data"], ["linear", "sigmoid", "poly", "rbf"]) self.assertTrue(project.hyperparameter["kernel"]["type"] is str) self.assertEqual(project.max_iterations, 300) self.assertEqual(project.param1, 1) self.assertEqual(project.param2, 2) self.assertEqual(project.function(2, 3), 5) self.assertTrue(project.get_typeof("C") is float) self.assertTrue(project.get_typeof("kernel") is str) project = HyppopyProject() project.add_hyperparameter(name="C", domain="uniform", data=[0.0001, 20], type=float) project.add_hyperparameter(name="kernel", domain="categorical", data=["linear", "sigmoid", "poly", "rbf"], type=str) self.assertEqual(project.hyperparameter["C"]["domain"], "uniform") self.assertEqual(project.hyperparameter["C"]["data"], [0.0001, 20]) self.assertTrue(project.hyperparameter["C"]["type"] is float) self.assertEqual(project.hyperparameter["kernel"]["domain"], "categorical") self.assertEqual(project.hyperparameter["kernel"]["data"], ["linear", "sigmoid", "poly", "rbf"]) self.assertTrue(project.hyperparameter["kernel"]["type"] is str) project.set_settings(max_iterations=500) self.assertEqual(project.max_iterations, 500) project.add_setting("my_param", 42) self.assertEqual(project.my_param, 42) project.add_setting("max_iterations", 200) self.assertEqual(project.max_iterations, 200)
def compute_deviation(solver_name, vfunc_id, iterations, N, fname): project = HyppopyProject() project.add_hyperparameter(name="axis_00", domain="uniform", data=[0, 1], type=float) project.add_hyperparameter(name="axis_01", domain="uniform", data=[0, 1], type=float) project.add_hyperparameter(name="axis_02", domain="uniform", data=[0, 1], type=float) project.add_hyperparameter(name="axis_03", domain="uniform", data=[0, 1], type=float) project.add_hyperparameter(name="axis_04", domain="uniform", data=[0, 1], type=float) vfunc = FunctionSimulator() vfunc.load_default(vfunc_id) minima = vfunc.minima() def my_loss_function(data, params): return vfunc(**params) blackbox = BlackboxFunction(data=[], blackbox_func=my_loss_function) results = {} results["gt"] = [] for mini in minima: results["gt"].append(np.median(mini[0])) for iter in iterations: results[iter] = { "minima": {}, "distance": {}, "duration": None, "set_difference": None, "loss": None, "loss_history": {} } for i in range(vfunc.dims()): results[iter]["minima"]["axis_0{}".format(i)] = [] results[iter]["distance"]["axis_0{}".format(i)] = [] project.add_setting("max_iterations", iter) project.add_setting("solver", solver_name) solver = SolverPool.get(project=project) solver.blackbox = blackbox axis_minima = [] best_losses = [] best_sets_diff = [] for i in range(vfunc.dims()): axis_minima.append([]) loss_history = [] durations = [] for n in range(N): print("\rSolver={} iteration={} round={}".format(solver, iter, n), end="") start = time.time() solver.run(print_stats=False) end = time.time() durations.append(end - start) df, best = solver.get_results() loss_history.append(np.flip(np.sort(df['losses'].values))) best_row = df['losses'].idxmin() best_losses.append(df['losses'][best_row]) best_sets_diff.append( abs(df['axis_00'][best_row] - best['axis_00']) + abs(df['axis_01'][best_row] - best['axis_01']) + abs(df['axis_02'][best_row] - best['axis_02']) + abs(df['axis_03'][best_row] - best['axis_03']) + abs(df['axis_04'][best_row] - best['axis_04'])) for i in range(vfunc.dims()): tmp = df['axis_0{}'.format(i)][best_row] axis_minima[i].append(tmp) results[iter]["loss_history"] = loss_history for i in range(vfunc.dims()): results[iter]["minima"]["axis_0{}".format(i)] = [ np.mean(axis_minima[i]), np.std(axis_minima[i]) ] dist = np.sqrt((axis_minima[i] - results["gt"][i])**2) results[iter]["distance"]["axis_0{}".format(i)] = [ np.mean(dist), np.std(dist) ] results[iter]["loss"] = [np.mean(best_losses), np.std(best_losses)] results[iter]["set_difference"] = sum(best_sets_diff) results[iter]["duration"] = np.mean(durations) file = open(fname, 'wb') pickle.dump(results, file) file.close()
def new_project(self, name="HyppopyProject", config=None): if name in self._projects.keys(): name = self.check_projectname(name) self._projects[name] = HyppopyProject(config) self._current_project = self._projects[name] return self._current_project
def compute_deviation(solver_name, vfunc_id, iterations, N, fname): project = HyppopyProject() project.add_hyperparameter(name="axis_00", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_01", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_02", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_03", domain="uniform", data=[0, 1], dtype="float") project.add_hyperparameter(name="axis_04", domain="uniform", data=[0, 1], dtype="float") vfunc = VirtualFunction() vfunc.load_default(vfunc_id) minima = vfunc.minima() def my_loss_function(data, params): return vfunc(**params) blackbox = BlackboxFunction(data=[], blackbox_func=my_loss_function) results = {} results["gt"] = [] for mini in minima: results["gt"].append(np.median(mini[0])) for iter in iterations: results[iter] = {"minima": {}, "loss": None} for i in range(vfunc.dims()): results[iter]["minima"]["axis_0{}".format(i)] = [] project.add_settings(section="solver", name="max_iterations", value=iter) project.add_settings(section="custom", name="use_solver", value=solver_name) solver = SolverPool.get(project=project) solver.blackbox = blackbox axis_minima = [] best_losses = [] for i in range(vfunc.dims()): axis_minima.append([]) for n in range(N): print("\rSolver={} iteration={} round={}".format(solver, iter, n), end="") solver.run(print_stats=False) df, best = solver.get_results() best_row = df['losses'].idxmin() best_losses.append(df['losses'][best_row]) for i in range(vfunc.dims()): tmp = df['axis_0{}'.format(i)][best_row] axis_minima[i].append(tmp) for i in range(vfunc.dims()): results[iter]["minima"]["axis_0{}".format(i)] = [np.mean(axis_minima[i]), np.std(axis_minima[i])] results[iter]["loss"] = [np.mean(best_losses), np.std(best_losses)] file = open(fname, 'wb') pickle.dump(results, file) file.close()
"kernel": { "domain": "categorical", "data": ["linear", "sigmoid", "poly", "rbf"], "type": "str" } }, "settings": { "solver": { "max_iterations": 500 }, "custom": {} }} # When creating a HyppopyProject instance we # pass the config dictionary to the constructor. project = HyppopyProject(config=config) # When building the project programmatically we can also use the methods # add_hyperparameter and add_settings project.clear() project.add_hyperparameter(name="C", domain="uniform", data=[0.0001, 20], dtype="float") project.add_hyperparameter(name="kernel", domain="categorical", data=["linear", "sigmoid"], dtype="str") project.add_settings(section="solver", name="max_iterations", value=500) # The custom section can be used freely project.add_settings(section="custom", name="my_var", value=10) # Settings are automatically transformed to member variables of the project class with the section as prefix if project.solver_max_iterations < 1000 and project.custom_my_var == 10: print("Project configured!")