def test_minima(self):
vfunc = FunctionSimulator()
vfunc.load_images(os.path.join(TESTDATA_DIR, 'functionsimulator'))
minima = vfunc.minima()
gt = [[[0.7265625], 0.48828125], [[-4.0234375], -7.890625],
[[2.265625], 0.859375],
[[
-17.421875, -17.353515625, -17.28515625, -17.216796875,
-17.1484375, -17.080078125, -17.01171875, -16.943359375,
-16.875, -16.806640625, -16.73828125, -16.669921875,
-16.6015625, -16.533203125, -16.46484375, -16.396484375,
-16.328125, -16.259765625, -16.19140625, -16.123046875,
-16.0546875, -15.986328125, -15.91796875, -15.849609375,
-15.78125, -15.712890625, -15.64453125, -15.576171875,
-15.5078125, -15.439453125, -15.37109375, -15.302734375,
-15.234375, -15.166015625, -15.09765625, -15.029296875,
-14.9609375, -14.892578125, -14.82421875, -14.755859375,
-14.6875, -14.619140625, -14.55078125, -14.482421875,
-14.4140625, -14.345703125, -14.27734375, -14.208984375,
-14.140625, -14.072265625, -14.00390625, -13.935546875,
-13.8671875, -13.798828125, -13.73046875, -13.662109375,
-13.59375, -13.525390625, -13.45703125, -13.388671875,
-13.3203125, -13.251953125, -13.18359375, -13.115234375,
-13.046875, -12.978515625, -12.91015625, -12.841796875,
-12.7734375, -12.705078125, -12.63671875, -12.568359375,
-12.5, -12.431640625, -12.36328125, -12.294921875,
-12.2265625, -12.158203125, -12.08984375, -12.021484375,
-11.953125, -11.884765625, -11.81640625, -11.748046875,
-11.6796875, -11.611328125, -11.54296875, -11.474609375,
-11.40625, -11.337890625, -11.26953125, -11.201171875,
-11.1328125, -11.064453125, -10.99609375, -10.927734375,
-10.859375, -10.791015625, -10.72265625, -10.654296875,
-10.5859375, -10.517578125, -10.44921875, -10.380859375,
-10.3125, -10.244140625, -10.17578125, -10.107421875,
-10.0390625, -9.970703125, -9.90234375, -9.833984375,
-9.765625, -9.697265625, -9.62890625, -9.560546875,
-9.4921875, -9.423828125, -9.35546875, -9.287109375,
-9.21875, -9.150390625, -9.08203125, -9.013671875,
-8.9453125, -8.876953125, -8.80859375, -8.740234375,
-8.671875, -8.603515625, -8.53515625, -8.466796875,
-8.3984375, -8.330078125, -8.26171875, -8.193359375, -8.125,
-8.056640625, -7.98828125, -7.919921875, -7.8515625,
-7.783203125, -7.71484375, -7.646484375, -7.578125,
-7.509765625, -7.44140625, -7.373046875, -7.3046875,
-7.236328125, -7.16796875, -7.099609375, -7.03125
], -9.125],
[[
5.44921875, 5.458984375, 5.46875, 5.478515625, 5.48828125,
5.498046875, 5.5078125, 5.517578125, 5.52734375
], 2.09375]]
self.assertAlmostEqual(minima, gt)
def test_imagereading(self):
vfunc = FunctionSimulator()
vfunc.load_images(os.path.join(TESTDATA_DIR, 'functionsimulator'))
self.assertTrue(isinstance(vfunc.data, np.ndarray))
self.assertEqual(vfunc.data.shape[0], 5)
self.assertEqual(vfunc.data.shape[1], 512)
gt = [
0.83984375 * 5, 0.44140625 * 20 - 10, 0.25390625 * 20,
0.81640625 * 8 - 10, 0.67578125 * 2 + 2
]
for i in range(5):
self.assertAlmostEqual(vfunc.data[i][0], gt[i])
gt = [[0, 1], [-10, 10], [0, 20], [-30, 5], [5, 10]]
for i in range(5):
self.assertEqual(vfunc.axis[i][0], gt[i][0])
self.assertEqual(vfunc.axis[i][1], gt[i][1])
def test_solver_loguniform(self):
config = {
"hyperparameter": {
"axis_00": {
"domain": "loguniform",
"data": [0.00001, 300],
"type": float,
"frequency": 21
},
"axis_01": {
"domain": "loguniform",
"data": [0.00001, 0.8],
"type": float,
"frequency": 21
},
"axis_02": {
"domain": "loguniform",
"data": [4, 6],
"type": float,
"frequency": 21
}
}
}
project = HyppopyProject(config)
solver = GridsearchSolver(project)
vfunc = FunctionSimulator()
vfunc.load_default()
solver.blackbox = vfunc
solver.run(print_stats=False)
df, best = solver.get_results()
self.assertAlmostEqual(best['axis_00'], 299.999, places=1)
self.assertAlmostEqual(best['axis_01'], 0.25869, places=1)
self.assertAlmostEqual(best['axis_02'], 5.10169, places=1)
for status in df['status']:
self.assertTrue(status)
for loss in df['losses']:
self.assertTrue(isinstance(loss, float))
def test_getOptunitySolver(self):
config = {
"hyperparameter": {
"axis_00": {
"domain": "uniform",
"data": [300, 800],
"type": float
},
"axis_01": {
"domain": "uniform",
"data": [-1, 1],
"type": float
},
"axis_02": {
"domain": "uniform",
"data": [0, 10],
"type": float
}
},
"max_iterations": 100
}
project = HyppopyProject(config)
solver = SolverPool.get("optunity", project)
self.assertTrue(isinstance(solver, OptunitySolver))
vfunc = FunctionSimulator()
vfunc.load_default()
solver.blackbox = vfunc
solver.run(print_stats=False)
df, best = solver.get_results()
self.assertTrue(300 <= best['axis_00'] <= 800)
self.assertTrue(-1 <= best['axis_01'] <= 1)
self.assertTrue(0 <= best['axis_02'] <= 10)
for status in df['status']:
self.assertTrue(status)
for loss in df['losses']:
self.assertTrue(isinstance(loss, float))
def test_solver_loguniform(self):
config = {
"hyperparameter": {
"axis_00": {
"domain": "normal",
"data": [500, 650],
"type": float
},
"axis_01": {
"domain": "loguniform",
"data": [0.001, 1],
"type": float
},
"axis_02": {
"domain": "normal",
"data": [4.5, 5.5],
"type": float
}
},
"max_iterations": 500,
}
project = HyppopyProject(config)
solver = HyperoptSolver(project)
vfunc = FunctionSimulator()
vfunc.load_default()
solver.blackbox = vfunc
solver.run(print_stats=False)
df, best = solver.get_results()
self.assertTrue(575 <= best['axis_00'] <= 585)
self.assertTrue(0.001 <= best['axis_01'] <= 1.0)
self.assertTrue(4.7 <= best['axis_02'] <= 5.3)
for status in df['status']:
self.assertTrue(status)
for loss in df['losses']:
self.assertTrue(isinstance(loss, float))
def test_solver_normal(self):
config = {
"hyperparameter": {
"axis_00": {
"domain": "normal",
"data": [500, 650],
"type": float
},
"axis_01": {
"domain": "normal",
"data": [0, 1],
"type": float
},
"axis_02": {
"domain": "normal",
"data": [4, 5],
"type": float
}
},
"max_iterations": 500,
}
solver = RandomsearchSolver(config)
vfunc = FunctionSimulator()
vfunc.load_default()
solver.blackbox = vfunc
solver.run(print_stats=False)
df, best = solver.get_results()
self.assertTrue(500 <= best['axis_00'] <= 650)
self.assertTrue(0 <= best['axis_01'] <= 1)
self.assertTrue(4 <= best['axis_02'] <= 5)
for status in df['status']:
self.assertTrue(status)
for loss in df['losses']:
self.assertTrue(isinstance(loss, float))
def test_data_adding(self):
gt = [[-10, 10], [-30, 5]]
vfunc = FunctionSimulator()
dim0 = np.arange(0, 1.1, 0.1)
dim1 = np.arange(1.0, -0.1, -0.1)
vfunc.add_dimension(dim0, gt[0])
self.assertEqual(len(vfunc.data.shape), 2)
self.assertEqual(vfunc.data.shape[0], 1)
self.assertEqual(vfunc.data.shape[1], 11)
vfunc.add_dimension(dim1, gt[1])
self.assertEqual(vfunc.data.shape[0], 2)
self.assertEqual(vfunc.data.shape[1], 11)
for n in range(11):
self.assertAlmostEqual(dim0[n], vfunc.data[0, n])
self.assertAlmostEqual(dim1[n], vfunc.data[1, n])
for i in range(2):
self.assertEqual(vfunc.axis[i][0], gt[i][0])
self.assertEqual(vfunc.axis[i][1], gt[i][1])
def compute_deviation(solver_name, vfunc_id, iterations, N, fname):
project = HyppopyProject()
project.add_hyperparameter(name="axis_00",
domain="uniform",
data=[0, 1],
type=float)
project.add_hyperparameter(name="axis_01",
domain="uniform",
data=[0, 1],
type=float)
project.add_hyperparameter(name="axis_02",
domain="uniform",
data=[0, 1],
type=float)
project.add_hyperparameter(name="axis_03",
domain="uniform",
data=[0, 1],
type=float)
project.add_hyperparameter(name="axis_04",
domain="uniform",
data=[0, 1],
type=float)
vfunc = FunctionSimulator()
vfunc.load_default(vfunc_id)
minima = vfunc.minima()
def my_loss_function(data, params):
return vfunc(**params)
blackbox = BlackboxFunction(data=[], blackbox_func=my_loss_function)
results = {}
results["gt"] = []
for mini in minima:
results["gt"].append(np.median(mini[0]))
for iter in iterations:
results[iter] = {
"minima": {},
"distance": {},
"duration": None,
"set_difference": None,
"loss": None,
"loss_history": {}
}
for i in range(vfunc.dims()):
results[iter]["minima"]["axis_0{}".format(i)] = []
results[iter]["distance"]["axis_0{}".format(i)] = []
project.add_setting("max_iterations", iter)
project.add_setting("solver", solver_name)
solver = SolverPool.get(project=project)
solver.blackbox = blackbox
axis_minima = []
best_losses = []
best_sets_diff = []
for i in range(vfunc.dims()):
axis_minima.append([])
loss_history = []
durations = []
for n in range(N):
print("\rSolver={} iteration={} round={}".format(solver, iter, n),
end="")
start = time.time()
solver.run(print_stats=False)
end = time.time()
durations.append(end - start)
df, best = solver.get_results()
loss_history.append(np.flip(np.sort(df['losses'].values)))
best_row = df['losses'].idxmin()
best_losses.append(df['losses'][best_row])
best_sets_diff.append(
abs(df['axis_00'][best_row] - best['axis_00']) +
abs(df['axis_01'][best_row] - best['axis_01']) +
abs(df['axis_02'][best_row] - best['axis_02']) +
abs(df['axis_03'][best_row] - best['axis_03']) +
abs(df['axis_04'][best_row] - best['axis_04']))
for i in range(vfunc.dims()):
tmp = df['axis_0{}'.format(i)][best_row]
axis_minima[i].append(tmp)
results[iter]["loss_history"] = loss_history
for i in range(vfunc.dims()):
results[iter]["minima"]["axis_0{}".format(i)] = [
np.mean(axis_minima[i]),
np.std(axis_minima[i])
]
dist = np.sqrt((axis_minima[i] - results["gt"][i])**2)
results[iter]["distance"]["axis_0{}".format(i)] = [
np.mean(dist), np.std(dist)
]
results[iter]["loss"] = [np.mean(best_losses), np.std(best_losses)]
results[iter]["set_difference"] = sum(best_sets_diff)
results[iter]["duration"] = np.mean(durations)
file = open(fname, 'wb')
pickle.dump(results, file)
file.close()
def my_loss_function(data, params):
vfunc = FunctionSimulator()
vfunc.load_default("5D")
return vfunc(**params)