def test_get_rbf_matrix(self):
"""Test basic properties of the RBF matrix (e.g. symmetry, size).
Verify that the RBF matrix is symmetric and it has the correct
size for all types of RBF.
"""
settings = RbfoptSettings()
for i in range(50):
dim = random.randint(1, 20)
num_points = random.randint(10, 50)
node_pos = np.array(
[[random.uniform(-100, 100) for j in range(dim)]
for k in range(num_points)])
# Possible shapes of the matrix
for rbf_type in self.rbf_types:
settings.rbf = rbf_type
mat = ru.get_rbf_matrix(settings, dim, num_points, node_pos)
self.assertIsInstance(mat, np.matrix)
self.assertAlmostEqual(np.max(mat - mat.transpose()),
0.0,
msg='RBF matrix is not symmetric')
size = num_points + 1
if (ru.get_degree_polynomial(settings) > 0):
size += dim**ru.get_degree_polynomial(settings)
self.assertEqual(mat.shape, (size, size))
# Check that exception is raised for unknown RBF types
settings.rbf = 'unknown'
self.assertRaises(ValueError, ru.get_rbf_matrix, settings, dim,
num_points, node_pos)
def test_get_model_quality_estimate(self):
"""Test the get_model_quality_estimate function.
"""
for rbf in ['cubic', 'thin_plate_spline', 'multiquadric', 'linear']:
settings = RbfoptSettings(rbf=rbf)
error = ru.get_model_quality_estimate(settings, self.n, self.k,
self.node_pos, self.node_val,
self.k)
# Create a copy of the interpolation nodes and values
sorted_idx = self.node_val.argsort()
sorted_node_val = self.node_val[sorted_idx]
# Initialize the arrays used for the cross-validation
cv_node_pos = self.node_pos[sorted_idx[1:]]
cv_node_val = self.node_val[sorted_idx[1:]]
# The node that was left out
rm_node_pos = self.node_pos[sorted_idx[0]]
rm_node_val = self.node_val[sorted_idx[0]]
# Estimate of the model error
loo_error = 0.0
for i in range(self.k):
# Compute the RBF interpolant with one node left out
Amat = ru.get_rbf_matrix(settings, self.n, self.k - 1,
cv_node_pos)
rbf_l, rbf_h = ru.get_rbf_coefficients(settings, self.n,
self.k - 1, Amat,
cv_node_val)
# Compute value of the interpolant at the removed node
predicted_val = ru.evaluate_rbf(settings, rm_node_pos, self.n,
self.k - 1, cv_node_pos, rbf_l,
rbf_h)
# Update leave-one-out error
loc = np.searchsorted(sorted_node_val, predicted_val)
loo_error += abs(loc - i)
# Update the node left out
if (i < self.k - 1):
tmp = cv_node_pos[i].copy()
cv_node_pos[i] = rm_node_pos
rm_node_pos = tmp
cv_node_val[i], rm_node_val = rm_node_val, cv_node_val[i]
self.assertAlmostEqual(loo_error,
error,
msg='Model selection procedure ' +
'miscomputed the error')
def rbf_optimize(settings, dimension, var_lower, var_upper, objfun,
objfun_fast = None, integer_vars = None,
init_node_pos = None, init_node_val = None,
output_stream = sys.stdout):
"""Optimize a black-box function.
Optimize an unknown function over a box using the enhanced RBF
method.
Parameters
----------
settings : rbfopt_settings.RbfSettings
Global and algorithmic settings.
dimension : int
The dimension of the problem, i.e. size of the space.
var_lower : List[float]
Vector of variable lower bounds.
var_upper : List[float]
Vector of variable upper bounds.
objfun : Callable[List[float]]
The unknown function we want to optimize.
objfun_fast : Callable[List[float]]
A faster, lower quality version of the unknown function we
want to optimize. If None, it is assumed that such a version
of the function is not available.
integer_vars : List[int] or None
A list containing the indices of the integrality constrained
variables. If None or empty list, all variables are assumed to
be continuous.
init_node_pos : List[List[float]] or None
Coordinates of points at which the function value is known. If
None, the initial points will be generated by the
algorithm. This must be of length at least dimension + 1, if
provided.
init_node_val : List[float] or None
Function values corresponding to the points given in
init_node_pos. Should be None if the previous argument is
None.
output_stream : file or None
A stream object that will be used to print output. By default,
this will be the standard output stream.
Returns
---
(float, List[float], int, int, int)
A quintuple (value, point, itercount, evalcount,
fast_evalcount) containing the objective function value of the
best solution found, the corresponding value of the decision
variables, the number of iterations of the algorithm, the
total number of function evaluations, and the number of these
evaluations that were performed in 'fast' mode.
"""
assert(len(var_lower) == dimension)
assert(len(var_upper) == dimension)
assert((integer_vars is None) or (len(integer_vars) == 0) or
(max(integer_vars) < dimension))
assert(init_node_pos is None or
(len(init_node_pos) == len(init_node_val) and
len(init_node_pos) >= dimension + 1))
assert(isinstance(settings, RbfSettings))
# Start timing
start_time = time.time()
# Set the value of 'auto' parameters if necessary
l_settings = settings.set_auto_parameters(dimension, var_lower,
var_upper, integer_vars)
# Local and global RBF models are usually the same
best_local_rbf, best_global_rbf = l_settings.rbf, l_settings.rbf
# We use n to denote the dimension of the problem, same notation
# of the paper. This is redundant but it simplifies our life.
n = dimension
# Set random seed. Some of the (external) libraries use numpy's
# random generator, we use python's internal generator, so we have
# to seed both for consistency.
random.seed(l_settings.rand_seed)
np.random.seed(l_settings.rand_seed)
# Iteration number
itercount = 0
# Total number of function evaluations in accurate mode
evalcount = 0
# Total number of fast function evaluation
fast_evalcount = 0
# Identifier of the current step within the cyclic optimization
# strategy counter. This typically increases at every iteration,
# but sometimes we may decide to repeat a step.
current_step = 0
# Current number of consecutive local searches
num_cons_ls = 0
# Number of consecutive cycles without improvement
num_stalled_cycles = 0
# Number of consecutive discarded points
num_cons_discarded = 0
# Number of restarts in fast mode
num_fast_restarts = 0
# Initialize identifiers of the search steps
inf_step = 0
local_search_step = (l_settings.num_global_searches + 1)
cycle_length = (l_settings.num_global_searches + 2)
restoration_step = (l_settings.num_global_searches + 3)
# Determine which step is the first of each loop
first_step = (inf_step if l_settings.do_infstep else inf_step + 1)
# Initialize settings for two-phase optimization.
# two_phase_optimization indicates if the fast buy noisy objective
# function is available.
# is_best_fast indicates if the best known objective function
# value was evaluated in fast mode or in accurate mode.
# current_mode indicates the evaluation mode for the objective
# function at a given stage.
if (objfun_fast is not None):
two_phase_optimization = True
is_best_fast = True
current_mode = 'fast'
else:
two_phase_optimization = False
is_best_fast = False
current_mode = 'accurate'
# Round variable bounds to integer if necessary
ru.round_integer_bounds(var_lower, var_upper, integer_vars)
# List of node coordinates is node_pos, list of node values is in
# node_val. We keep a current list and a global list; they can be
# different in case of restarts.
# We must choose the initial interpolation points. If they are not
# given, generate them using the chosen strategy.
if (init_node_pos is None):
node_pos = ru.initialize_nodes(l_settings, var_lower, var_upper,
integer_vars)
if (current_mode == 'accurate'):
node_val = [objfun(point) for point in node_pos]
evalcount += len(node_val)
else:
node_val = [objfun_fast(point) for point in node_pos]
fast_evalcount += len(node_val)
node_is_fast = [current_mode == 'fast' for val in node_val]
else:
node_pos = init_node_pos
node_val = init_node_val
# We assume that initial points provided by the user are
# 'accurate'.
node_is_fast = [False for val in node_val]
# Make a copy, in the original space
all_node_pos = [point for point in node_pos]
all_node_val = [val for val in node_val]
# Store if each function evaluation is fast or accurate
all_node_is_fast = [val for val in node_is_fast]
# We need to remember the index of the first node in all_node_pos
# after every restart
all_node_pos_size_at_restart = 0
# Rescale the domain of the function
node_pos = [ru.transform_domain(l_settings, var_lower, var_upper, point)
for point in node_pos]
(l_lower,
l_upper) = ru.transform_domain_bounds(l_settings, var_lower, var_upper)
# Current minimum value among the nodes, and its index
fmin_index = node_val.index(min(node_val))
fmin = node_val[fmin_index]
# Current maximum value among the nodes
fmax = max(all_node_val)
# Denominator of errormin
gap_den = (abs(l_settings.target_objval)
if (abs(l_settings.target_objval) >= l_settings.eps_zero)
else 1.0)
# Shift due to fast function evaluation
gap_shift = (ru.get_fast_error_bounds(l_settings, fmin)[1]
if is_best_fast else 0.0)
# Current minimum distance from the optimum
gap = ((fmin + gap_shift - l_settings.target_objval)/gap_den)
# Best value function at the beginning of an optimization cycle
fmin_cycle_start = fmin
# Print the initialization points
for (i, val) in enumerate(node_val):
min_dist = ru.get_min_distance(node_pos[i], node_pos[:i] +
node_pos[(i+1):])
print('Iteration {:3d}'.format(itercount) +
' {:16s}'.format('Initialization') +
': objval{:s}'.format('~' if node_is_fast[i] else ' ') +
' {:16.6f}'.format(val) +
' min_dist {:9.4f}'.format(min_dist) +
' gap {:8.2f}'.format(gap*100),
file = output_stream)
# Main loop
while (itercount < l_settings.max_iterations and
evalcount < l_settings.max_evaluations and
time.time() - start_time < l_settings.max_clock_time and
gap > l_settings.eps_opt):
# If the user wants to skip inf_step as in the original paper
# of Gutmann (2001), we proceed to the next iteration.
if (current_step == inf_step and not l_settings.do_infstep):
current_step = (current_step+1) % cycle_length
continue
# Check if we should restart. We only restart if the initial
# sampling strategy is random, otherwise it makes little sense.
if (num_cons_discarded >= l_settings.max_consecutive_discarded or
(num_stalled_cycles >= l_settings.max_stalled_cycles and
evalcount + n + 1 < l_settings.max_evaluations and
l_settings.init_strategy != 'all_corners' and
l_settings.init_strategy != 'lower_corners')):
print('Restart at iteration {:3d}'.format(itercount),
file = output_stream)
output_stream.flush()
# We update the number of fast restarts here, so that if
# we hit the limit on fast restarts, we can evaluate
# points in accurate mode after restarting (even if
# current_mode is updated in a subsequent block of code)
num_fast_restarts += (1 if current_mode == 'fast' else 0.0)
# Store the current number of nodes
all_node_pos_size_at_restart = len(all_node_pos)
# Compute a new set of starting points
node_pos = ru.initialize_nodes(l_settings, var_lower, var_upper,
integer_vars)
if (current_mode == 'accurate' or
num_fast_restarts > l_settings.max_fast_restarts or
fast_evalcount + n + 1 >= l_settings.max_fast_evaluations):
node_val = [objfun(point) for point in node_pos]
evalcount += len(node_val)
else:
node_val = [objfun_fast(point) for point in node_pos]
fast_evalcount += len(node_val)
node_is_fast = [current_mode == 'fast' for val in node_val]
# Print the initialization points
for (i, val) in enumerate(node_val):
min_dist = ru.get_min_distance(node_pos[i], node_pos[:i] +
node_pos[(i+1):])
print('Iteration {:3d}'.format(itercount) +
' {:16s}'.format('Initialization') +
': objval{:s}'.format('~' if node_is_fast[i] else ' ') +
' {:16.6f}'.format(val) +
' min_dist {:9.4f}'.format(min_dist) +
' gap {:8.2f}'.format(gap*100),
file = output_stream)
all_node_pos.extend(node_pos)
all_node_val.extend(node_val)
all_node_is_fast.extend(node_is_fast)
# Rescale the domain of the function
node_pos = [ru.transform_domain(l_settings, var_lower,
var_upper, point)
for point in node_pos]
(l_lower, l_upper) = ru.transform_domain_bounds(l_settings,
var_lower,
var_upper)
# Update all counters and values to restart properly
fmin_index = node_val.index(min(node_val))
fmin = node_val[fmin_index]
fmax = max(node_val)
fmin_cycle_start = fmin
num_stalled_cycles = 0
num_cons_discarded = 0
is_best_fast = node_is_fast[fmin_index]
# Number of nodes at current iteration
k = len(node_pos)
# Compute indices of fast node evaluations (sparse format)
fast_node_index = ([i for (i, val) in enumerate(node_is_fast) if val]
if two_phase_optimization else list())
# If function scaling is automatic, determine which one to use
if (settings.function_scaling == 'auto' and
current_step <= first_step):
sorted_node_val = sorted(node_val)
if (sorted_node_val[len(sorted_node_val)//2] -
sorted_node_val[0] > l_settings.log_scaling_threshold):
l_settings.function_scaling = 'log'
else:
l_settings.function_scaling = 'off'
# Rescale nodes if necessary
(scaled_node_val, scaled_fmin, scaled_fmax,
node_err_bounds) = ru.transform_function_values(l_settings, node_val,
fmin, fmax,
fast_node_index)
# If RBF selection is automatic, at the beginning of each
# cycle check if a different RBF yields a better model
if (settings.rbf == 'auto' and k > n+1 and current_step <= first_step):
best_local_rbf = ms.get_best_rbf_model(l_settings, n, k, node_pos,
scaled_node_val,
int(math.ceil(k*0.1)))
best_global_rbf = ms.get_best_rbf_model(l_settings, n, k, node_pos,
scaled_node_val,
int(math.ceil(k*0.7)))
# If we are in local search or just before local search, use a
# local model.
if (current_step >= (local_search_step - 1)):
l_settings.rbf = best_local_rbf
# Otherwise, global.
else:
l_settings.rbf = best_global_rbf
try:
# Compute the matrices necessary for the algorithm
Amat = ru.get_rbf_matrix(l_settings, n, k, node_pos)
Amatinv = ru.get_matrix_inverse(l_settings, Amat)
# Compute RBF interpolant at current stage
if (fast_node_index):
# Get coefficients for the exact RBF
rc = ru.get_rbf_coefficients(l_settings, n, k,
Amat, scaled_node_val)
# RBF with some fast function evaluations
rc = aux.get_noisy_rbf_coefficients(l_settings, n, k,
Amat[:k, :k],
Amat[:k, k:],
scaled_node_val,
fast_node_index,
node_err_bounds,
rc[0], rc[1])
(rbf_l, rbf_h) = rc
else:
# Fully accurate RBF
rc = ru.get_rbf_coefficients(l_settings, n, k,
Amat, scaled_node_val)
(rbf_l, rbf_h) = rc
except np.linalg.LinAlgError:
# Error in the solution of the linear system. We must
# switch to a restoration phase.
current_step = restoration_step
node_val.pop()
node_pos.pop()
if (node_is_fast.pop()):
fast_node_index.pop()
(scaled_node_val, scaled_fmin, scaled_fmax,
node_err_bounds) = ru.transform_function_values(l_settings,
node_val,
fmin, fmax,
fast_node_index)
k = len(node_pos)
# For displaying purposes, record what type of iteration we
# are performing
iteration_id = ''
# Initialize the new point to None
next_p = None
if (current_step == inf_step):
# Infstep: explore the parameter space
next_p = aux.maximize_one_over_mu(l_settings, n, k, l_lower,
l_upper, node_pos, Amatinv,
integer_vars)
iteration_id = 'InfStep'
elif (current_step == restoration_step):
# Restoration: pick a random point, far enough from
# current points
restoration_done = False
cons_restoration = 0
while (not restoration_done and
cons_restoration < l_settings.max_consecutive_restoration):
next_p = [random.uniform(var_lower[i], var_upper[i])
for i in range(n)]
ru.round_integer_vars(next_p, integer_vars)
if (ru.get_min_distance(next_p, node_pos) >
l_settings.min_dist):
# Try inverting the RBF matrix to see if
# nonsingularity is restored
try:
Amat = ru.get_rbf_matrix(l_settings, n, k + 1,
node_pos + [next_p])
Amatinv = ru.get_matrix_inverse(l_settings, Amat)
restoration_done = True
except np.linalg.LinAlgError:
cons_restoration += 1
else:
cons_restoration += 1
if (not restoration_done):
print('Restoration phase keeps failing. Abort.',
file = output_stream)
# This will force the optimization process to return
break
iteration_id = 'Restoration'
elif (current_step == local_search_step):
# Local search: compute the minimum of the RBF.
min_rbf = aux.minimize_rbf(l_settings, n, k, l_lower,
l_upper, node_pos,
rbf_l, rbf_h, integer_vars)
if (min_rbf is not None):
min_rbf_val = ru.evaluate_rbf(l_settings, min_rbf, n, k,
node_pos, rbf_l, rbf_h)
# If the RBF cannot me minimized, or if the minimum is
# larger than the node with smallest value, just take the
# node with the smallest value.
if (min_rbf is None or
(min_rbf_val >= scaled_fmin + l_settings.eps_zero)):
min_rbf = node_pos[fmin_index]
min_rbf_val = scaled_fmin
# Check if point can be accepted: is there an improvement?
if (min_rbf_val <= (scaled_fmin - l_settings.eps_impr *
max(1.0, abs(scaled_fmin)))):
target_val = min_rbf_val
next_p = min_rbf
iteration_id = 'LocalStep'
else:
# If the point is not improving, we solve a global
# search problem, rescaling the search box to enforce
# some sort of local search
target_val = scaled_fmin - 0.01*max(1.0, abs(scaled_fmin))
local_varl = [max(l_lower[i], min_rbf[i] -
l_settings.local_search_box_scaling *
0.33 * (l_upper[i] - l_lower[i]))
for i in range(n)]
local_varu = [min(l_upper[i], min_rbf[i] +
l_settings.local_search_box_scaling *
0.33 * (l_upper[i] - l_lower[i]))
for i in range(n)]
ru.round_integer_bounds(local_varl, local_varu,
integer_vars)
next_p = aux.maximize_h_k(l_settings, n, k, local_varl,
local_varu, node_pos, rbf_l,
rbf_h, Amatinv, target_val,
integer_vars)
iteration_id = 'AdjLocalStep'
else:
# Global search: compromise between finding a good value
# of the objective function, and improving the model.
# Choose target value for the objective function. To do
# so, we need the minimum of the RBF interpolant.
min_rbf = aux.minimize_rbf(l_settings, n, k, l_lower,
l_upper, node_pos,
rbf_l, rbf_h, integer_vars)
if (min_rbf is not None):
min_rbf_val = ru.evaluate_rbf(l_settings, min_rbf, n, k,
node_pos, rbf_l, rbf_h)
# If the RBF cannot me minimized, or if the minimum is
# larger than the node with smallest value, just take the
# node with the smallest value.
if (min_rbf is None or
min_rbf_val >= scaled_fmin + l_settings.eps_zero):
min_rbf = node_pos[fmin_index]
min_rbf_val = scaled_fmin
# The scaling factor is 1 - h/kappa, where h goes from
# 0 to kappa-1 over the course of one global search
# cycle, and kappa is the number of global searches.
scaling = (1 - ((current_step - 1) /
l_settings.num_global_searches))**2
# Compute the function value used to determine the target
# value. This is given by the sorted value in position
# sigma_n, where sigma_n is a function described in the
# paper by Gutmann (2001). If clipping is disabled, we
# simply take the largest function value.
if (l_settings.targetval_clipping):
local_fmax = ru.get_fmax_current_iter(l_settings, n, k,
current_step,
scaled_node_val)
else:
local_fmax = fmax
target_val = (min_rbf_val -
scaling * (local_fmax - min_rbf_val))
# If the global search is almost a local search, we
# restrict the search to a box following Regis and
# Shoemaker (2007)
if (scaling <= config.LOCAL_SEARCH_THRESHOLD):
local_varl = [max(l_lower[i], min_rbf[i] -
l_settings.local_search_box_scaling *
math.sqrt(scaling) *
(l_upper[i] - l_lower[i]))
for i in range(n)]
local_varu = [min(l_upper[i], min_rbf[i] +
l_settings.local_search_box_scaling *
math.sqrt(scaling) *
(l_upper[i] - l_lower[i]))
for i in range(n)]
ru.round_integer_bounds(local_varl, local_varu,
integer_vars)
# Otherwise, use original bounds
else:
local_varl = l_lower
local_varu = l_upper
next_p = aux.maximize_h_k(l_settings, n, k, local_varl,
local_varu, node_pos, rbf_l,
rbf_h, Amatinv, target_val,
integer_vars)
iteration_id = 'GlobalStep'
# -- end if
# If previous points were evaluated in low quality and we are
# now in high-quality local search mode, then we should verify
# if it is better to evaluate a brand new point or re-evaluate
# a previously known point.
if ((two_phase_optimization == True) and
(current_step == local_search_step) and
(current_mode == 'accurate')):
(ind, bump) = ru.get_min_bump_node(l_settings, n, k, Amat,
scaled_node_val,
fast_node_index,
node_err_bounds, target_val)
if (ind is not None and next_p is not None):
# Check if the newly proposed point is very close to
# an existing one.
if (ru.get_min_distance(next_p, node_pos) >
l_settings.min_dist):
# If not, compute bumpiness of the newly proposed point.
n_bump = ru.get_bump_new_node(l_settings, n, k, node_pos,
scaled_node_val, next_p,
fast_node_index,
node_err_bounds, target_val)
else:
# If yes, we will simply reevaluate the existing
# point (if it can be reevaluated).
ind = ru.get_min_distance_index(next_p, node_pos)
n_bump = (float('inf') if node_is_fast[ind]
else float('-inf'))
if (n_bump > bump):
# In this case we want to put the new point at the
# same location as one of the old points. Remove
# the noisy function evaluation from the data
# structures, so that the point can be evaluated
# in accurate mode.
next_p = node_pos.pop(ind)
node_val.pop(ind)
node_is_fast.pop(ind)
all_node_pos.pop(all_node_pos_size_at_restart + ind)
all_node_val.pop(all_node_pos_size_at_restart + ind)
all_node_is_fast.pop(all_node_pos_size_at_restart + ind)
# We must update k here to make sure it is consistent
# until the start of the next iteration.
k = len(node_pos)
# If the optimization failed or the point is too close to
# current nodes, discard it. Otherwise, add it to the list.
if ((next_p is None) or
(ru.get_min_distance(next_p, node_pos) <= l_settings.min_dist)):
current_step = (current_step+1) % cycle_length
num_cons_ls = 0
num_cons_discarded += 1
print('Iteration {:3d}'.format(itercount) + ' Discarded',
file = output_stream)
output_stream.flush()
else:
min_dist = ru.get_min_distance(next_p, node_pos)
# Transform back to original space if necessary
next_p_orig = ru.transform_domain(l_settings, var_lower,
var_upper, next_p, True)
# Evaluate the new point, in accurate mode or fast mode
if (current_mode == 'accurate'):
next_val = objfun(next_p_orig)
evalcount += 1
node_is_fast.append(False)
else:
next_val = objfun_fast(next_p_orig)
fast_evalcount += 1
# Check if the point improves over existing points, or
# if it could be optimal according to tolerances. In
# this case, perform a double evaluation.
best_possible = fmin + (ru.get_fast_error_bounds(l_settings,
fmin)[0]
if is_best_fast else 0.0)
if ((next_val <= best_possible -
l_settings.eps_impr*max(1.0, abs(best_possible))) or
(next_val <= l_settings.target_objval +
l_settings.eps_opt*abs(l_settings.target_objval) -
ru.get_fast_error_bounds(l_settings, next_val)[0])):
print('Iteration {:3d}'.format(itercount) +
' {:16s}'.format(iteration_id) + ': objval~' +
' {:16.6f}'.format(next_val) +
' min_dist {:9.4f}'.format(min_dist) +
' gap {:8.2f}'.format(gap*100),
file = output_stream)
output_stream.flush()
next_val = objfun(next_p_orig)
evalcount += 1
node_is_fast.append(False)
else:
node_is_fast.append(True)
# Add to the lists
node_pos.append(next_p)
node_val.append(next_val)
all_node_pos.append(next_p_orig)
all_node_val.append(next_val)
all_node_is_fast.append(node_is_fast[-1])
if ((current_step == local_search_step) and
(next_val <= fmin - l_settings.eps_impr*max(1.0,abs(fmin))) and
(num_cons_ls < l_settings.max_consecutive_local_searches - 1)):
# Keep doing local search
num_cons_ls += 1
else:
current_step = (current_step+1) % cycle_length
num_cons_ls = 0
num_cons_discarded = 0
# Update fmin
if (next_val < fmin):
fmin_index = k
fmin = next_val
is_best_fast = node_is_fast[-1]
fmax = max(fmax, next_val)
# Shift due to fast function evaluation
gap_shift = (ru.get_fast_error_bounds(l_settings, next_val)[1]
if is_best_fast else 0.0)
gap = min(gap, (next_val + gap_shift - l_settings.target_objval) /
gap_den)
print('Iteration {:3d}'.format(itercount) +
' {:16s}'.format(iteration_id) +
': objval{:s}'.format('~' if node_is_fast[-1] else ' ') +
' {:16.6f}'.format(next_val) +
' min_dist {:9.4f}'.format(min_dist) +
' gap {:8.2f}'.format(gap*100),
file = output_stream)
output_stream.flush()
# Update iteration number
itercount += 1
# At the beginning of each loop of the cyclic optimization
# strategy, check if the main loop is stalling
if (current_step <= first_step):
if (fmin <= (fmin_cycle_start - l_settings.max_stalled_objfun_impr
* max(1.0, abs(fmin_cycle_start)))):
num_stalled_cycles = 0
fmin_cycle_start = fmin
else:
num_stalled_cycles += 1
# Check if we should switch to the second phase of two-phase
# optimization. The conditions for switching are:
# 1) Optimization in fast mode restarted too many times.
# 2) We reached the limit of fast mode iterations.
if ((two_phase_optimization == True) and (current_mode == 'fast') and
((num_fast_restarts > l_settings.max_fast_restarts) or
(itercount >= l_settings.max_fast_iterations) or
(fast_evalcount >= l_settings.max_fast_evaluations))):
print('Switching to accurate mode ' +
'at iteration {:3d}'.format(itercount),
file = output_stream)
output_stream.flush()
current_mode = 'accurate'
# -- end while
# Find best point and return it
i = all_node_val.index(min(all_node_val))
fmin = all_node_val[i]
gap_shift = (ru.get_fast_error_bounds(l_settings, fmin)[1]
if all_node_is_fast[i] else 0.0)
gap = ((fmin + gap_shift - l_settings.target_objval) / gap_den)
print('Summary: iterations {:3d}'.format(itercount) +
' evaluations {:3d}'.format(evalcount) +
' fast_evals {:3d}'.format(fast_evalcount) +
' clock time {:7.2f}'.format(time.time() - start_time) +
' objval{:s}'.format('~' if (all_node_is_fast[i]) else ' ') +
' {:15.6f}'.format(fmin) +
' gap {:6.2f}'.format(100*gap),
file = output_stream)
output_stream.flush()
return (all_node_val[i], all_node_pos[i],
itercount, evalcount, fast_evalcount)
def get_model_quality_estimate_clp(settings, n, k, node_pos, node_val,
num_iterations):
"""Compute an estimate of model quality using LPs with Clp.
Computes an estimate of model quality, performing
cross-validation. This version does not use all interpolation
nodes, but rather only the best num_iterations ones, and it uses a
sequence of LPs instead of a brute-force calculation. It should be
much faster than the brute-force calculation, as each LP in the
sequence requires only two pivots. The solver is COIN-OR Clp.
Parameters
----------
settings : rbfopt_settings.RbfSettings
Global and algorithmic settings.
n : int
Dimension of the problem, i.e. the space where the point lives.
k : int
Number of nodes, i.e. interpolation points.
node_pos : List[List[float]]
Location of current interpolation nodes.
node_val : List[float]
List of values of the function at the nodes.
num_iterations : int
Number of nodes on which quality should be tested.
Returns
-------
float
An estimate of the leave-one-out cross-validation error, which
can be interpreted as a measure of model quality.
Raises
------
RuntimeError
If the LPs cannot be solved.
ValueError
If some settings are not supported.
"""
assert(isinstance(settings, RbfSettings))
assert(len(node_val)==k)
assert(len(node_pos)==k)
assert(num_iterations <= k)
assert(cpx_available)
# We cannot find a leave-one-out interpolant if the following
# condition is not met.
assert(k > n + 1)
# Get size of polynomial part of the matrix (p) and sign of obj
# function (sign)
if (ru.get_degree_polynomial(settings) == 1):
p = n + 1
sign = 1
elif (ru.get_degree_polynomial(settings) == 0):
p = 1
sign = -1
else:
raise ValueError('RBF type ' + settings.rbf + ' not supported')
# Sort interpolation nodes by increasing objective function value
sorted_list = sorted([(node_val[i], node_pos[i]) for i in range(k)])
# Initialize the arrays used for the cross-validation
cv_node_pos = [sorted_list[i][1] for i in range(k)]
cv_node_val = [sorted_list[i][0] for i in range(k)]
# Compute the RBF matrix, and the two submatrices of interest
Amat = ru.get_rbf_matrix(settings, n, k, cv_node_pos)
Phimat = Amat[:k, :k]
Pmat = Amat[:k, k:]
identity = np.matrix(np.identity(k))
# Instantiate model
model = CyLPModel()
# Add variables: lambda, h, slacks
rbf_lambda = model.addVariable('rbf_lambda', k)
rbf_h = model.addVariable('rbf_h', p)
slack = model.addVariable('slack', k)
# Add constraints: interpolation conditions, unisolvence
F = CyLPArray(cv_node_val)
zeros = CyLPArray([0] * p)
ones = CyLPArray([1] * p)
if (p > 1):
# We can use matrix form
model += (Phimat * rbf_lambda + Pmat * rbf_h + identity * slack == F)
model += (Pmat.transpose() * rbf_lambda == zeros)
else:
# Workaround for one dimensional variables: one row at a time
for i in range(k):
model += Phimat[i, :] * rbf_lambda + rbf_h + slack[i] == F[i]
model += (Pmat.transpose() * rbf_lambda == zeros)
# Add objective
obj = CyLPArray([sign*val for val in node_val])
model.objective = obj * rbf_lambda
# Create LP, suppress output, and set bounds
lp = clp.CyClpSimplex(model)
lp.logLevel = 0
infinity = lp.getCoinInfinity()
for i in range(k + p):
lp.setColumnLower(i, -infinity)
lp.setColumnUpper(i, infinity)
for i in range(k):
lp.setColumnLower(k + p + i, 0.0)
lp.setColumnUpper(k + p + i, 0.0)
# Estimate of the model error
loo_error = 0.0
for i in range(num_iterations):
# Fix lambda[i] to zero
lp.setColumnLower(i, 0.0)
lp.setColumnUpper(i, 0.0)
# Set slack[i] to unconstrained
lp.setColumnLower(k + p + i, -infinity)
lp.setColumnUpper(k + p + i, infinity)
lp.dual(startFinishOptions = 'sx')
if (not lp.getStatusCode() == 0):
raise RuntimeError('Could not solve LP with Clp, status ' +
lp.getStatusString())
lambda_sol = lp.primalVariableSolution['rbf_lambda'].tolist()
h_sol = lp.primalVariableSolution['rbf_h'].tolist()
# Compute value of the interpolant at the removed node
predicted_val = ru.evaluate_rbf(settings, cv_node_pos[i], n, k,
cv_node_pos, lambda_sol, h_sol)
# Update leave-one-out error
loo_error += abs(predicted_val - cv_node_val[i])
# Fix lambda[i] to zero
lp.setColumnLower(i, -infinity)
lp.setColumnUpper(i, infinity)
# Set slack[i] to unconstrained
lp.setColumnLower(k + p + i, 0.0)
lp.setColumnUpper(k + p + i, 0.0)
return loo_error/num_iterations
def get_model_quality_estimate_full(settings, n, k, node_pos, node_val):
"""Compute an estimate of model quality.
Computes an estimate of model quality, performing
cross-validation. This version assumes that all interpolation
points are used for cross-validation.
Parameters
----------
settings : rbfopt_settings.RbfSettings
Global and algorithmic settings.
n : int
Dimension of the problem, i.e. the space where the point lives.
k : int
Number of nodes, i.e. interpolation points.
node_pos : List[List[float]]
Location of current interpolation nodes.
node_val : List[float]
List of values of the function at the nodes.
Returns
-------
float
An estimate of the leave-one-out cross-validation error, which
can be interpreted as a measure of model quality.
"""
assert(isinstance(settings, RbfSettings))
assert(len(node_val)==k)
assert(len(node_pos)==k)
# We cannot find a leave-one-out interpolant if the following
# condition is not met.
assert(k > n + 1)
sorted_list = sorted([(node_val[i], node_pos[i]) for i in range(k)])
# Create a copy of the interpolation nodes and values
cv_node_pos = [node_pos[i] for i in range(k-1)]
cv_node_val = [node_val[i] for i in range(k-1)]
# The node that was left out
rm_node_pos = node_pos[-1]
rm_node_val = node_val[-1]
# Estimate of the model error
loo_error = 0.0
# Variance
loo_error_var = 0.0
for i in range(k):
# Compute the RBF interpolant with one node left out
Amat = ru.get_rbf_matrix(settings, n, k-1, cv_node_pos)
(rbf_l, rbf_h) = ru.get_rbf_coefficients(settings, n, k-1,
Amat, cv_node_val)
# Compute value of the interpolant at the removed node
predicted_val = ru.evaluate_rbf(settings, rm_node_pos, n, k-1,
cv_node_pos, rbf_l, rbf_h)
# Update leave-one-out error
loo_error += abs(predicted_val - rm_node_val)
loo_error_var += (abs(predicted_val - rm_node_val))**2
# Update the node left out, unless we are at the last iteration
if (i < k - 1):
cv_node_pos[k-2-i], rm_node_pos = rm_node_pos, cv_node_pos[k-2-i]
cv_node_val[k-2-i], rm_node_val = rm_node_val, cv_node_val[k-2-i]
return loo_error/k
def get_model_quality_estimate_cpx(settings, n, k, node_pos, node_val,
num_iterations):
"""Compute an estimate of model quality using LPs with Cplex.
Computes an estimate of model quality, performing
cross-validation. This version does not use all interpolation
nodes, but rather only the best num_iterations ones, and it uses a
sequence of LPs instead of a brute-force calculation. It should be
much faster than the brute-force calculation, as each LP in the
sequence requires only two pivots. The solver is IBM-ILOG Cplex.
Parameters
----------
settings : rbfopt_settings.RbfSettings
Global and algorithmic settings.
n : int
Dimension of the problem, i.e. the space where the point lives.
k : int
Number of nodes, i.e. interpolation points.
node_pos : List[List[float]]
Location of current interpolation nodes.
node_val : List[float]
List of values of the function at the nodes.
num_iterations : int
Number of nodes on which quality should be tested.
Returns
-------
float
An estimate of the leave-one-out cross-validation error, which
can be interpreted as a measure of model quality.
Raises
------
RuntimeError
If the LPs cannot be solved.
ValueError
If some settings are not supported.
"""
assert(isinstance(settings, RbfSettings))
assert(len(node_val)==k)
assert(len(node_pos)==k)
assert(num_iterations <= k)
assert(cpx_available)
# We cannot find a leave-one-out interpolant if the following
# condition is not met.
assert(k > n + 1)
# Get size of polynomial part of the matrix (p) and sign of obj
# function (sign)
if (ru.get_degree_polynomial(settings) == 1):
p = n + 1
sign = 1
elif (ru.get_degree_polynomial(settings) == 0):
p = 1
sign = -1
else:
raise ValueError('RBF type ' + settings.rbf + ' not supported')
# Sort interpolation nodes by increasing objective function value
sorted_list = sorted([(node_val[i], node_pos[i]) for i in range(k)])
# Initialize the arrays used for the cross-validation
cv_node_pos = [sorted_list[i][1] for i in range(k)]
cv_node_val = [sorted_list[i][0] for i in range(k)]
# Compute the RBF matrix
Amat = ru.get_rbf_matrix(settings, n, k, cv_node_pos)
# Instantiate model and suppress output
lp = cpx.Cplex()
lp.set_log_stream(None)
lp.set_error_stream(None)
lp.set_warning_stream(None)
lp.set_results_stream(None)
lp.cleanup(settings.eps_zero)
lp.parameters.threads.set(1)
# lp.parameters.simplex.display.set(2)
# Add variables: lambda, h, slacks
lp.variables.add(obj = [sign*val for val in node_val],
lb = [-cpx.infinity] * k,
ub = [cpx.infinity] * k)
lp.variables.add(lb = [-cpx.infinity] * p,
ub = [cpx.infinity] * p)
lp.variables.add(lb = [0] * k,
ub = [0] * k)
# Add constraints: interpolation conditions, unisolvence
expr = [cpx.SparsePair(ind = [j for j in range(k + p)] + [k + p + i],
val = [Amat[i, j] for j in range(k + p)] + [1.0])
for i in range(k)]
lp.linear_constraints.add(lin_expr = expr, senses = ['E'] * k,
rhs = cv_node_val)
expr = [cpx.SparsePair(ind = [j for j in range(k + p)],
val = [Amat[i, j] for j in range(k + p)])
for i in range(k, k + p)]
lp.linear_constraints.add(lin_expr = expr, senses = ['E'] * p,
rhs = [0] * p)
# Estimate of the model error
loo_error = 0.0
for i in range(num_iterations):
# Fix lambda[i] to zero
lp.variables.set_lower_bounds(i, 0.0)
lp.variables.set_upper_bounds(i, 0.0)
# Set slack[i] to unconstrained
lp.variables.set_lower_bounds(k + p + i, -cpx.infinity)
lp.variables.set_upper_bounds(k + p + i, cpx.infinity)
lp.solve()
if (not lp.solution.is_primal_feasible()):
raise RuntimeError('Could not solve LP with Cplex')
rbf_l = lp.solution.get_values(0, k - 1)
rbf_h = lp.solution.get_values(k, k + p - 1)
# Compute value of the interpolant at the removed node
predicted_val = ru.evaluate_rbf(settings, cv_node_pos[i], n, k,
cv_node_pos, rbf_l, rbf_h)
# Update leave-one-out error
loo_error += abs(predicted_val - cv_node_val[i])
# Fix lambda[i] to zero
lp.variables.set_lower_bounds(i, -cpx.infinity)
lp.variables.set_upper_bounds(i, cpx.infinity)
# Set slack[i] to unconstrained
lp.variables.set_lower_bounds(k + p + i, 0.0)
lp.variables.set_upper_bounds(k + p + i, 0.0)
return loo_error/num_iterations
def get_model_quality_estimate(settings, n, k, node_pos, node_val,
num_iterations):
"""Compute an estimate of model quality.
Computes an estimate of model quality, performing
cross-validation. This version does not use all interpolation
nodes, but rather only the best num_iterations ones.
Parameters
----------
settings : rbfopt_settings.RbfSettings
Global and algorithmic settings.
n : int
Dimension of the problem, i.e. the space where the point lives.
k : int
Number of nodes, i.e. interpolation points.
node_pos : List[List[float]]
Location of current interpolation nodes.
node_val : List[float]
List of values of the function at the nodes.
num_iterations : int
Number of nodes on which quality should be tested.
Returns
-------
float
An estimate of the leave-one-out cross-validation error, which
can be interpreted as a measure of model quality.
"""
assert(isinstance(settings, RbfSettings))
assert(len(node_val)==k)
assert(len(node_pos)==k)
assert(num_iterations <= k)
# We cannot find a leave-one-out interpolant if the following
# condition is not met.
assert(k > n + 1)
# Sort interpolation nodes by increasing objective function value
sorted_list = sorted([(node_val[i], node_pos[i]) for i in range(k)])
# Initialize the arrays used for the cross-validation
cv_node_pos = [sorted_list[i][1] for i in range(1,k)]
cv_node_val = [sorted_list[i][0] for i in range(1,k)]
# The node that was left out
rm_node_pos = sorted_list[0][1]
rm_node_val = sorted_list[0][0]
# Estimate of the model error
loo_error = 0.0
for i in range(num_iterations):
# Compute the RBF interpolant with one node left out
Amat = ru.get_rbf_matrix(settings, n, k-1, cv_node_pos)
(rbf_l, rbf_h) = ru.get_rbf_coefficients(settings, n, k-1,
Amat, cv_node_val)
# Compute value of the interpolant at the removed node
predicted_val = ru.evaluate_rbf(settings, rm_node_pos, n, k-1,
cv_node_pos, rbf_l, rbf_h)
# Update leave-one-out error
loo_error += abs(predicted_val - rm_node_val)
# Update the node left out
cv_node_pos[i], rm_node_pos = rm_node_pos, cv_node_pos[i]
cv_node_val[i], rm_node_val = rm_node_val, cv_node_val[i]
return loo_error/num_iterations
