def test_global_function_search():
spec_F = dlib.function_spec([-10,-10], [10,10])
spec_G = dlib.function_spec([-2], [6])
opt = dlib.global_function_search([spec_F, spec_G])
for i in range(15):
next = opt.get_next_x()
#print("next x is for function {} and has coordinates {}".format(next.function_idx, next.x))
if (next.function_idx == 0):
a = next.x[0]
b = next.x[1]
next.set(F(a,b))
else:
x = next.x[0]
next.set(G(x))
[x,y,function_idx] = opt.get_best_function_eval()
#print("\nbest function was {}, with y of {}, and x of {}".format(function_idx,y,x))
assert(abs(y-2) < 1e-7)
assert(abs(x[0]-5) < 1e-7)
assert(function_idx==1)
def test_global_function_search():
spec_F = dlib.function_spec([-10, -10], [10, 10])
spec_G = dlib.function_spec([-2], [6])
opt = dlib.global_function_search([spec_F, spec_G])
for i in range(15):
next = opt.get_next_x()
#print("next x is for function {} and has coordinates {}".format(next.function_idx, next.x))
if (next.function_idx == 0):
a = next.x[0]
b = next.x[1]
next.set(F(a, b))
else:
x = next.x[0]
next.set(G(x))
[x, y, function_idx] = opt.get_best_function_eval()
#print("\nbest function was {}, with y of {}, and x of {}".format(function_idx,y,x))
assert (abs(y - 2) < 1e-7)
assert (abs(x[0] - 5) < 1e-7)
assert (function_idx == 1)
def _load(fname):
"""
Load a pickle file containing
(spec, results, info) where
results: np.array of shape [N, M+1] where
N is number of trials
M is number of hyperparameters
results[:, 0] is result/loss
results[:, 1:] is [param1, param2, ...]
spec: (is_integer, lower, upper)
where each element is list of length M
info: dict with keys
params, solver_epsilon, relative_noise_magnitude, pp
Assumes only 1 function is optimized over
Returns
(dlib.function_spec, [dlib.function_eval], dict, prev_best)
where prev_best: np.array[result, param1, param2, ...]
"""
raw_spec, raw_results, info = _load_raw(fname)
is_integer, lo_bounds, hi_bounds = raw_spec
spec = dlib.function_spec(bound1=lo_bounds,
bound2=hi_bounds,
is_integer=is_integer)
evals = []
prev_best = raw_results[np.argmax(raw_results, axis=0)[0]]
for raw_result in raw_results:
x = list(raw_result[1:])
result = dlib.function_evaluation(x=x, y=raw_result[0])
evals.append(result)
return raw_spec, spec, evals, info, prev_best
def __init__(self,
target_fn,
space,
maximize=True,
max_parallel=5,
kwargs={}):
super().__init__(target_fn, space, maximize, max_parallel, kwargs)
lowers = [x[0] for x in space.transformed_bounds]
uppers = [x[1] for x in space.transformed_bounds]
constraints = dlib.function_spec(lowers, uppers)
self.solver = dlib.global_function_search(constraints)
def dlib_optimizer(logger, obj_func, initial_theta, bounds):
"""
dlib GFS optimizer for optimizing hyper parameters of GPR
Input:
* 'obj_func' is the objective function to be minimized, which
takes the hyperparameters theta as parameter and an
optional flag eval_gradient, which determines if the
gradient is returned additionally to the function value
* 'initial_theta': the initial value for theta, which can be
used by local optimizers
* 'bounds': the bounds on the values of theta,
[(lb1, ub1), (lb2, ub2), (lb3, ub3)]
Returned:
* 'theta_opt' is the best found hyperparameters theta
* 'func_min' is the corresponding value of the target function.
"""
import dlib
nopt = len(bounds)
is_int = []
lb = []
ub = []
for i, bd in enumerate(bounds):
lb.append(bd[0])
ub.append(bd[1])
is_int.append(type(bd[0]) == int and type(bd[1]) == int)
spec = dlib.function_spec(bound1=lb, bound2=ub, is_integer=is_int)
progress_frac = 100
maxn = 1000
eps = 0.01
optimizer = dlib.global_function_search([spec])
optimizer.set_solver_epsilon(eps)
for i in range(maxn):
logger.info(f'GPR optimization loop: {i} iterations: ')
next_eval = optimizer.get_next_x()
next_eval.set(-obj_func(list(next_eval.x))[0])
if (i > 0) and (i % progress_frac == 0):
best_eval = optimizer.get_best_function_eval()
logger.info(f'GPR optimization loop: {i} iterations: ')
logger.info(
f' best evaluation so far: {-best_eval[1]} @ {list(best_eval[0])}'
)
best_eval = optimizer.get_best_function_eval()
theta_opt = np.asarray(list(best_eval[0]))
func_min = -best_eval[1]
return theta_opt, func_min
def find_best_parameters():
# set up the optimizer
func_spec = dlib.function_spec(
bound1=[lo for lo, hi, is_int in HYPERPARAM_RANGES.values()],
bound2=[hi for lo, hi, is_int in HYPERPARAM_RANGES.values()],
is_integer=[is_int for lo, hi, is_int in HYPERPARAM_RANGES.values()],
)
optimizer = dlib.global_function_search(
functions=[func_spec],
initial_function_evals=[[]],
relative_noise_magnitude=0.001,
)
# if the optimizer expects that refining the current best solution won't result
# in an improvement of at least this amount, then it will explore elsewhere
optimizer.set_solver_epsilon(100) # in score units
# main eval/solver loop
best_score = -math.inf
while True:
next_eval_request = optimizer.get_next_x()
if PRINT_ALL:
print('Evaluating with parameters:')
print(list(next_eval_request.x))
print(list(next_eval_request.x))
print(list(next_eval_request.x))
for name, value in zip(HYPERPARAM_RANGES.keys(), next_eval_request.x):
print(f' {name}: {value}')
median_score = test_hyperparams(*next_eval_request.x)
next_eval_request.set(median_score)
if median_score > best_score:
best_score = median_score
print('New best median score:', best_score)
print(f' Params: {dict(zip(HYPERPARAM_RANGES.keys(), next_eval_request.x))}')
print()
# check if stdin has input and stop if so
# if select.select([sys.stdin,],[],[],0.0)[0]:
# # termios.tcflush(sys.stdin, termios.TCIOFLUSH) # flush stdin to prevent keypresses from going to the shell
# break
best_hyperparams, best_score, func_idx = optimizer.get_best_function_eval()
# print the best hyperparameters
print()
print('Best average score:', best_score)
print('Parameters:')
for name, value in zip(HYPERPARAM_RANGES.keys(), best_hyperparams):
print(f' {name}: {value}')
print()
def adalipo_search(mc,
func,
num_iterations,
mins,
maxs,
is_integer,
initial_values=[],
history_file="hp_opt_history.txt"):
"""
This function searches for the global maxima of func inside the
parameters.
Args:
mc:
func: Objective function to be optimized, defined as func(*args)
num_iterations: Number of iterations till maxima is found
mins[i]: minimum of hyperparameter #i to be optimized
mins[i]: maximum of hyperparameter #i to be optimized
is_integer[i]: if hyperparameter #i accepts only integer values
initial_values: if this optimization has been done for some iterations
in the past then the results can bee added here as a list
with entries of type dlib.function_evaluation.
Returns:
The best values of the hyperparameters
The best evaluation
"""
specs = dlib.function_spec(mins, maxs, is_integer)
if initial_values != []:
opt = dlib.global_function_search([specs] * len(initial_values),
initial_values, 0.001)
else:
opt = dlib.global_function_search(specs)
for _ in xrange(num_iterations):
function_evaluation_request_next = opt.get_next_x()
x = function_evaluation_request_next.x
f_val = func(*x)
function_evaluation_request_next.set(f_val)
with open(os.path.join(mc["BASE_DIR"], history_file), "a") as f:
y = list(x)
y.append(f_val)
f.write(str(y))
f.write("\n")
# return best point so far
r = opt.get_best_function_eval()
opt_hp = list(r[0])
opt_eval = r[1]
return (opt_hp, opt_eval)
def apply(self, problem, iterations):
X = problem.getDomain()
minValues = np.min(X, axis=1).tolist()
maxValues = np.max(X, axis=1).tolist()
def fWrapper2(*args):
return problem.evaluate(np.array(args))
spec_F = dlib.function_spec(minValues, maxValues)
opt = dlib.global_function_search(spec_F)
opt.set_seed(random.randint(0, 10000000))
opt.set_solver_epsilon(100000) #NO TR SETTING
for i in range(iterations):
next = opt.get_next_x()
next.set(fWrapper2(*next.x))
return problem
def _init_search(self):
function_spec = dlib.function_spec(
[self.lower_bounds[name] for name in self.arg_names],
[self.upper_bounds[name] for name in self.arg_names],
[self.is_integer[name] for name in self.arg_names],
)
self.search = dlib.global_function_search(
functions=[function_spec],
initial_function_evals=[[
dlib.function_evaluation(
[x[0][name] for name in self.arg_names], x[1])
for x in self._raw_evaluations
]],
relative_noise_magnitude=0.001,
)
self.search.set_solver_epsilon(self.epsilon)
if self.random_state is not None:
self.search.set_seed(self.random_state)
def __init__(self,
pp=None,
space=None,
solver_epsilon=None,
relative_noise_magnitude=None,
fname=None,
save=False):
"""
`Global Function Search
<http://dlib.net/optimization.html#global_function_search>`_
(GFS) Optimizer. Creates a GFS optimizer for
optimizing a set of hyperparameters. Supports parallel optimization
runs and averaging of stochastic optimization runs, as well as
saving/restoring both settings and progress to file.
Specify 'pp'+'space' and/or 'fname'.
If 'fname' is given, attempt to restore progress and settings from file.
If restoring fails, continue with specified/default settings
i.e. ``(pp, space, solver_epsilon, relative_noise_magnitude)``.
If restoring succeeds, then any setting passed as argument in addition
to the file name will be compared to the setting restored from the file,
and you will be given the choice of which one to use.
:param dict pp: Problem parameters.
All hyperparameters and their values for the objective
function, including those not being optimized over. E.g: ``{'beta': 0.44}``.
Can be an empty dict.
Can include hyperparameters being optimized over, but does not need to.
If a hyperparameter is specified in both 'pp' and 'space', its value
in 'pp' will be overridden.
:param dict space: Hyperparameters to optimize over.
Entries should be of the form:
``parameter: (Low_Bound, High_Bound)`` e.g:
``{'alpha': (0.65, 0.85), 'gamma': (1, 8)}``. If both bounds for a
parameter are Ints, then only integers within the (inclusive) range
will be sampled and tested.
:param str fname: File name for restoring and/or saving results,
progress and settings.
:param bool save: (optional) Save settings and progress periodically,
on user quit (CTRL-C), and on completion.
:param float solver_epsilon: (optional) The accuracy to which local optima
are determined before global exploration is resumed.
See `Dlib <http://dlib.net/dlib/global_optimization/
global_function_search_abstract.h.html#global_function_search>`_
for further documentation. Default: 0.0005
:param float relative_noise_magnitude: (optional) Should be increased for
highly stochastic objective functions. Deterministic and continuous
functions can use a value of 0. See `Dlib
<http://dlib.net/dlib/global_optimization/upper_bound_function_abstract.h.html
#upper_bound_function>`_
for further documentation. Default: 0.001
"""
# Verify inputs
if fname is None:
if pp is None or space is None:
raise ValueError(
"You must specify at least file name `fname` or problem "
"parameters `pp` along with a hyperparameter space `space`."
)
if save:
raise ValueError(
"If you want to save you must specify a file name `fname`."
)
else:
if not os.path.isfile(fname):
if pp is None or space is None:
raise FileNotFoundError(fname)
eps = solver_epsilon
noise_mag = relative_noise_magnitude
params, is_int, lo_bounds, hi_bounds = [], [], [], []
if space is not None:
for parm, conf in space.items():
params.append(parm)
lo, hi = conf
is_int.append(type(lo) == int and type(hi) == int)
lo_bounds.append(lo)
hi_bounds.append(hi)
old_evals = []
if fname is not None:
try:
# Load progress and settings from file, then compare each
# restored setting with settings specified by args (if any)
old_raw_spec, old_spec, old_evals, info, prev_best = _load(
fname)
saved_params = info['params']
print(
f"Restored {len(old_evals)} trials, prev best: "
f"{prev_best[0]}@{list(zip(saved_params, prev_best[1:]))}")
if params and params != saved_params:
# Switching params being optimized over would throw off Dlib.
# Must use restore params from specified
print(
f"Saved params {saved_params} differ from currently specified "
f"{params}. Using saved.")
params = saved_params
if is_int:
raw_spec = _cmp_and_choose('bounds', old_raw_spec,
(is_int, lo_bounds, hi_bounds))
else:
raw_spec = old_raw_spec
is_int, lo_bounds, hi_bounds = raw_spec
if len(params) != len(is_int):
raise ValueError(
f"Params {params} and spec {raw_spec} are of different length"
)
eps = _cmp_and_choose('solver_epsilon', info['solver_epsilon'],
eps)
noise_mag = _cmp_and_choose('relative_noise_magnitude',
info['relative_noise_magnitude'],
noise_mag)
_, pp = _compare_pps(info['pp'], pp)
except FileNotFoundError:
# Create a new file
pass
eps = 0.0005 if eps is None else eps
noise_mag = 0.001 if noise_mag is None else noise_mag
spec = dlib.function_spec(bound1=lo_bounds,
bound2=hi_bounds,
is_integer=is_int)
if old_evals:
optimizer = dlib.global_function_search(
[spec],
initial_function_evals=[old_evals],
relative_noise_magnitude=noise_mag)
else:
optimizer = dlib.global_function_search([spec])
optimizer.set_relative_noise_magnitude(noise_mag)
optimizer.set_solver_epsilon(eps)
self.pp, self.params, self.optimizer, self.spec = pp, params, optimizer, spec
self.eps, self.noise_mag, self.is_int = eps, noise_mag, is_int
self.fname, self.save = fname, save
def run(self):
pp = self.pp
logger = self.logger
# n_concurrent = cpu_count() - 4 # Number of concurrent procs
n_concurrent = 12
n_avg = 4
assert n_concurrent % n_avg == 0, \
f"n_avg {n_avg} does not evenly divide n_concurrent {n_concurrent}"
n_step = n_concurrent // n_avg
n_sims = 1000 # The number of times to sample and test params
save_iter = 30
eps = 0.0005 # solver_epsilon
noise_mag = 0.005 # relative_noise_magnitude. Default setting: 0.001
fname = "dlib-" + pp['hopt_fname'].replace('.pkl', '') + '.pkl'
space = {
# parameter: [IsInteger, Low_Bound, High_Bound]
'gamma': [False, 0.60, 0.95],
'lambda': [False, 0.60, 0.99],
'net_lr': [False, 5e-8, 8e-6],
'beta': [True, 10, 3000],
'net_lr_decay': [False, 0.70, 0.85],
'weight_beta': [False, 1e-3, 1e-1],
'weight_beta_decay': [False, 1e-5, 6e-5],
'grad_beta': [False, 5e-9, 5e-5],
'grad_beta_decay': [False, 1e-4, 1e-3],
'epsilon': [False, 2.5, 5],
'epsilon_decay': [False, 0.999_8, 0.999_999],
'alpha': [False, 0.00001, 0.3],
'huber_loss': [True, 10, 40]
}
space = {param: space[param] for param in pp['dlib_hopt']}
params, is_int, lo_bounds, hi_bounds = [], [], [], []
for p, li in space.items():
params.append(p)
is_int.append(li[0])
lo_bounds.append(li[1])
hi_bounds.append(li[2])
try:
old_raw_spec, old_spec, old_evals, info, prev_best = dlib_load(fname)
saved_params = info['params']
logger.error(f"Restored {len(old_evals)} trials, prev best: "
f"{prev_best[0]}@{list(zip(saved_params, prev_best[1:]))}")
# Switching params being optimized over would throw off DLIB.
# Have to assert that they are equal instead.
# What happens if you introduce another variable in addition to the previously?
# E.g. initialize dlib with evals over (eps, beta) then specify bounds for
# (eps, beta, gamma)?
# Restore saved params and settings if they differ from current/specified
if params != saved_params:
logger.error(
f"Saved params {saved_params} differ from currently specified "
f"{params}. Using saved.")
params = saved_params
raw_spec = cmp_and_choose('bounds', old_raw_spec,
(is_int, lo_bounds, hi_bounds))
spec = dlib.function_spec(
bound1=raw_spec[1], bound2=raw_spec[2], is_integer=raw_spec[0])
eps = cmp_and_choose('solver_epsilon', info['solver_epsilon'], eps)
noise_mag = cmp_and_choose('relative_noise_magnitude',
info['relative_noise_magnitude'], noise_mag)
_, pp = compare_pps(info['pp'], pp)
optimizer = dlib.global_function_search(
[spec],
initial_function_evals=[old_evals],
relative_noise_magnitude=noise_mag)
except FileNotFoundError:
spec = dlib.function_spec(
bound1=lo_bounds, bound2=hi_bounds, is_integer=is_int)
optimizer = dlib.global_function_search(spec)
optimizer.set_relative_noise_magnitude(noise_mag)
optimizer.set_solver_epsilon(eps)
# Becomes populated with results as simulations finished
result_queue = Queue()
simproc = partial(dlib_proc, self.stratclass, pp, params, result_queue)
# Becomes populated with evaluation objects to be set later
evals = [None] * n_sims
# Becomes populates with losses. When n_avg losses for a particular
# set of params are ready, their mean is set for the correponding eval.
results = [[] for _ in range(n_sims)]
def save_evals():
"""Store results of finished evals to file; print best eval"""
finished_evals = optimizer.get_function_evaluations()[1][0]
dlib_save(spec, finished_evals, params, eps, noise_mag, pp, fname)
best_eval = optimizer.get_best_function_eval()
prms = list(zip(params, list(best_eval[0])))
logger.error(f"Saving {len(finished_evals)} trials to {fname}."
f"Best eval so far: {best_eval[1]}@{prms}")
def spawn_evals(i):
"""Spawn a new sim process"""
eeval = optimizer.get_next_x()
evals[i] = eeval # Store eval object to be set with result later
vals = list(eeval.x)
logger.error(f"T{i} Testing {params}: {vals}")
for _ in range(n_avg):
Process(target=simproc, args=(i, vals)).start()
def store_result():
"""Block until a result is ready, then store it and report it to dlib"""
try:
# Blocks until a result is ready
i, result = result_queue.get()
except KeyboardInterrupt:
inp = ""
while inp not in ["Y", "N"]:
inp = input("Premature exit. Save? Y/N: ").upper()
if inp == "Y":
save_evals()
sys.exit(0)
else:
if result is not None:
results[i].append(result)
if len(results[i]) == n_avg:
evals[i].set(np.mean(results[i]))
if i > 0 and i % save_iter == 0 and len(results[i]) == n_avg:
save_evals()
""" the search will only attempt to find a global minimizer to at most
solver_epsilon accuracy. Once a local minimizer is found to that
accuracy the search will focus entirely on finding other minima
elsewhere rather than on further improving the current local optima
found so far. That is, once a local minima is identified to about
solver_epsilon accuracy, the algorithm will spend all its time
exploring the functions to find other local minima to investigate. An
epsilon of 0 means it will keep solving until it reaches full floating
point precision. Larger values will cause it to switch to pure global
exploration sooner and therefore might be more effective if your
objective function has many local minima and you don't care about a
super high precision solution.
On even iterations we pick the next x according to our upper bound while
on odd iterations we pick the next x according to the trust region model
"""
logger.error(f"Dlib hopt for {n_sims} sims with {n_concurrent} procs"
f" taking the average of {n_avg} runs"
f" on params {space} and s.eps {eps}")
# Spawn initial processes
for i in range(n_step):
spawn_evals(i)
# When a thread returns a result, start a new sim
for i in range(n_step, n_sims):
for _ in range(n_avg):
store_result()
spawn_evals(i)
# Get remaining results
for _ in range(n_step):
for _ in range(n_avg):
store_result()
save_evals()
def h5_load_all(file_path, opt_id):
"""
Loads an HDF5 file containing
(spec, results, info) where
results: np.array of shape [N, M+1] where
N is number of trials
M is number of hyperparameters
results[:, 0] is result/loss
results[:, 1:] is [param1, param2, ...]
spec: (is_integer, lower, upper)
where each element is list of length M
info: dict with keys
params, solver_epsilon, relative_noise_magnitude, problem
Assumes the structure is located in group /{opt_id}
Returns
(dlib.function_spec, [dlib.function_eval], dict, prev_best)
where prev_best: np.array[result, param1, param2, ...]
"""
raw_spec, raw_problem_results, info = h5_load_raw(file_path, opt_id)
is_integer, lo_bounds, hi_bounds = raw_spec
feature_names = info['features']
constraint_names = info['features']
spec = dlib.function_spec(bound1=lo_bounds,
bound2=hi_bounds,
is_integer=is_integer)
evals = {problem_id: [] for problem_id in raw_problem_results}
n_features = 0
feature_evals = None
if feature_names is not None:
n_features = len(feature_names)
feature_evals = {problem_id: [] for problem_id in raw_problem_results}
n_constraints = 0
constraint_evals = None
if constraint_names is not None:
n_constraints = len(constraint_names)
constraint_evals = {
problem_id: []
for problem_id in raw_problem_results
}
prev_best_dict = {}
for problem_id in raw_problem_results:
raw_results = raw_problem_results[problem_id]
ys = raw_results['objectives']['y']
xs = raw_results['parameters']
fs = None
cs = None
if n_features > 0:
fs = raw_results['features']
if n_constraints > 0:
cs = raw_results['constraints']
prev_best_index = np.argmax(ys, axis=0)
prev_best_dict[problem_id] = (ys[prev_best_index], xs[prev_best_index])
for i in range(ys.shape[0]):
x = list(xs[i])
result = dlib.function_evaluation(x=x, y=ys[i])
evals[problem_id].append(result)
if fs is not None:
feature_evals[problem_id].append(fs[i])
if cs is not None:
constraint_evals[problem_id].append(cs[i])
return raw_spec, spec, evals, feature_evals, constraint_evals, info, prev_best_dict
def __init__(self,
opt_id,
obj_fun,
reduce_fun=None,
problem_ids=None,
problem_parameters=None,
space=None,
feature_dtypes=None,
constraint_names=None,
solver_epsilon=None,
relative_noise_magnitude=None,
seed=None,
n_iter=100,
n_max_tasks=-1,
save_iter=10,
file_path=None,
save=False,
metadata=None,
verbose=False,
**kwargs):
"""`Creates an optimizer based on the Global Function Search
<http://dlib.net/optimization.html#global_function_search>`_
(GFS) optimizer in dlib. Supports distributed optimization
runs via mpi4py. Based on GFSOPtimizer by https://github.com/tsoernes
:param string opt_id: optimization group id
An identifier to associate with this class of optimization runs.
:param func obj_fun: function to maximize.
Must take as argument every parameter specified in
both 'problem_parameters' and 'space', in addition to 'pid',
and return the result as float.
'pid' specifies simulation run number.
If you want to minimize instead,
simply negate the result in the objective function before returning it.
:param set problem_ids (optional): Set of problem ids.
For solving sets of related problems with the same set of parameters.
If this parameter is not None, it is expected that the objective function
will return a dictionary of the form { problem_id: value }
:param dict problem_parameters: Problem parameters.
All hyperparameters and their values for the objective
function, including those not being optimized over. E.g: ``{'beta': 0.44}``.
Can be an empty dict.
Can include hyperparameters being optimized over, but does not need to.
If a hyperparameter is specified in both 'problem_parameters' and 'space', its value
in 'problem_parameters' will be overridden.
:param dict space: Hyperparameters to optimize over.
Entries should be of the form:
``parameter: (Low_Bound, High_Bound)`` e.g:
``{'alpha': (0.65, 0.85), 'gamma': (1, 8)}``. If both bounds for a
parameter are Ints, then only integers within the (inclusive) range
will be sampled and tested.
:param func reduce_fun: function to reduce multiple results per evaluation obtained from each distributed worker.
Must take as argument a list of objective evaluations.
:param int n_iter: (optional) Number of times to sample and test params.
:param int save_iter: (optional) How often to save progress.
:param str file_path: (optional) File name for restoring and/or saving results and settings.
:param bool save: (optional) Save settings and progress periodically.
:param float solver_epsilon: (optional) The accuracy to which local optima
are determined before global exploration is resumed.
See `Dlib <http://dlib.net/dlib/global_optimization/
global_function_search_abstract.h.html#global_function_search>`_
for further documentation. Default: 0.0005
:param float relative_noise_magnitude: (optional) Should be increased for
highly stochastic objective functions. Deterministic and continuous
functions can use a value of 0. See `Dlib
<http://dlib.net/dlib/global_optimization/upper_bound_function_abstract.h.html
#upper_bound_function>`_
for further documentation. Default: 0.001
:param float seed: (optional) Sets the seed used for random
sampling by the optimization algorithm. If None, the optimizer will always
produce the same deterministic behavior. Default: None
"""
self.opt_id = opt_id
self.verbose = verbose
self.logger = logging.getLogger(opt_id)
if self.verbose:
self.logger.setLevel(logging.INFO)
# Verify inputs
if file_path is None:
if problem_parameters is None or space is None:
raise ValueError(
"You must specify at least file name `file_path` or problem "
"parameters `problem_parameters` along with a hyperparameter space `space`."
)
if save:
raise ValueError(
"If you want to save you must specify a file name `file_path`."
)
else:
if not os.path.isfile(file_path):
if problem_parameters is None or space is None:
raise FileNotFoundError(file_path)
eps = solver_epsilon
noise_mag = relative_noise_magnitude
param_names, is_int, lo_bounds, hi_bounds = [], [], [], []
if space is not None:
for parm, conf in space.items():
param_names.append(parm)
lo, hi = conf
is_int.append(type(lo) == int and type(hi) == int)
lo_bounds.append(lo)
hi_bounds.append(hi)
old_evals = {}
if file_path is not None:
if os.path.isfile(file_path):
old_evals, old_feature_evals, old_constraint_evals, param_names, feature_dtypes, constraint_names, is_int, lo_bounds, hi_bounds, eps, noise_mag, problem_parameters, problem_ids = \
init_from_h5(file_path, param_names, opt_id, self.logger)
self.feature_dtypes = feature_dtypes
self.feature_names = None
if feature_dtypes is not None:
self.feature_names = [dtype[0] for dtype in feature_dtypes]
self.constraint_names = constraint_names
eps = 0.0005 if eps is None else eps
noise_mag = 0.001 if noise_mag is None else noise_mag
spec = dlib.function_spec(bound1=lo_bounds,
bound2=hi_bounds,
is_integer=is_int)
has_problem_ids = (problem_ids is not None)
if not has_problem_ids:
problem_ids = set([0])
n_saved_evals = 0
n_saved_features = 0
n_saved_constraints = 0
optimizer_dict = {}
for problem_id in problem_ids:
if problem_id in old_evals:
optimizer = dlib.global_function_search(
[spec],
initial_function_evals=[old_evals[problem_id]],
relative_noise_magnitude=noise_mag)
n_saved_evals = len(old_evals[problem_id])
else:
optimizer = dlib.global_function_search([spec])
optimizer.set_relative_noise_magnitude(noise_mag)
optimizer.set_solver_epsilon(eps)
if seed is not None:
optimizer.set_seed(seed)
optimizer_dict[problem_id] = optimizer
self.optimizer_dict = optimizer_dict
self.metadata = metadata
self.problem_parameters, self.param_names, self.spec = problem_parameters, param_names, spec
self.eps, self.noise_mag, self.is_int = eps, noise_mag, is_int
self.file_path, self.save = file_path, save
self.n_iter = n_iter
self.n_max_tasks = n_max_tasks
self.n_saved_evals = n_saved_evals
self.n_saved_features = n_saved_features
self.n_saved_constraints = n_saved_constraints
self.save_iter = save_iter
if has_problem_ids:
self.eval_fun = partial(eval_obj_fun_mp, obj_fun,
self.problem_parameters, self.param_names,
self.is_int, problem_ids)
else:
self.eval_fun = partial(eval_obj_fun_sp, obj_fun,
self.problem_parameters, self.param_names,
self.is_int, 0)
self.reduce_fun = reduce_fun
self.evals = {problem_id: {} for problem_id in problem_ids}
self.feature_evals = None
if self.feature_names is not None:
self.feature_evals = {problem_id: [] for problem_id in problem_ids}
self.constraint_evals = None
if self.constraint_names is not None:
self.constraint_evals = {
problem_id: []
for problem_id in problem_ids
}
self.has_problem_ids = has_problem_ids
self.problem_ids = problem_ids
# Problem parameters to be used in simulation and to be optimizer over
space = {'epsilon': [True, 10, 2000], 'alpha': [False, 0.00001, 0.3]}
params, is_int, lo_bounds, hi_bounds = [], [], [], []
for p, li in space.items():
params.append(p)
is_int.append(li[0])
lo_bounds.append(li[1])
hi_bounds.append(li[2])
n_sims = 4 # The number of times to sample and test params
n_concurrent = 4 # Number of concurrent procs
assert n_sims >= n_concurrent
spec = dlib.function_spec(bound1=lo_bounds,
bound2=hi_bounds,
is_integer=is_int)
optimizer = dlib.global_function_search(spec)
result_queue = Queue()
simproc = partial(worker_proc, Simulation, pp, params, result_queue)
results = np.zeros((n_sims, len(params) + 1))
evals = [None] * n_sims
logger.error(f"Dlib hopt for {n_sims} sims with {n_concurrent} procs for"
f" on params {params} with bounds {lo_bounds}, {hi_bounds}")
# Spawn initial processes
for i in range(n_concurrent):
eeval = optimizer.get_next_x()
next_x = list(eeval.x)
evals[i] = eeval
results[i][1:] = next_x