def _fit_poly(X, y):
try:
N, d = X.shape
myParameters = []
for dimension in range(d):
values = X[:,dimension]
values_min = np.amin(values)
values_max = np.amax(values)
if (values_min - values_max) ** 2 < 0.01:
myParameters.append(Parameter(distribution='Uniform', lower=values_min-0.01, upper=values_max+0.01, order=self.order))
else:
myParameters.append(Parameter(distribution='Uniform', lower=values_min, upper=values_max, order=self.order))
if self.basis == "hyperbolic-basis":
myBasis = Basis(self.basis, orders=[self.order for _ in range(d)], q=0.5)
else:
myBasis = Basis(self.basis, orders=[self.order for _ in range(d)])
container["index_node_global"] += 1
poly = Poly(myParameters, myBasis, method=self.poly_method, sampling_args={'sample-points':X, 'sample-outputs':y}, solver_args=self.poly_solver_args)
poly.set_model()
mse = np.linalg.norm(y - poly.get_polyfit(X).reshape(-1)) ** 2 / N
except Exception as e:
print("Warning fitting of Poly failed:", e)
print(d, values_min, values_max)
mse, poly = np.inf, None
return mse, poly
def get_subspace_polynomial(self):
"""
Returns a polynomial defined over the dimension reducing subspace.
:param Subspaces self:
An instance of the Subspaces object.
:return:
**subspacepoly**: A Poly object that defines a polynomial over the subspace. The distribution of parameters is
assumed to be uniform and the maximum and minimum bounds for each parameter are defined by the maximum and minimum values
of the project samples.
"""
active_subspace = self._subspace[:, 0:self.subspace_dimension]
projected_points = np.dot(self.sample_points, active_subspace)
myparameters = []
for i in range(0, self.subspace_dimension):
param = Parameter(distribution='uniform', lower=np.min(projected_points[:,i]), upper=np.max(projected_points[:,i]), \
order=self.polynomial_degree)
myparameters.append(param)
mybasis = Basis("total-order")
subspacepoly = Poly(myparameters, mybasis, method=self.poly_method, sampling_args={'sample-points':projected_points, \
'sample-outputs':self.sample_outputs},
solver_args=self.solver_args)
subspacepoly.set_model()
return subspacepoly
def vandermonde(eta, p):
"""
Internal function to variable_projection
Calculates the Vandermonde matrix using polynomial basis functions
:param eta: ndarray, the affine transformed projected values of inputs in active subspace
:param p: int, the maximum degree of polynomials
:return:
* **V (numpy array)**: The resulting Vandermode matrix
* **Polybasis (Poly object)**: An instance of Poly object containing the polynomial basis derived
"""
_, n = eta.shape
listing = []
for i in range(0, n):
listing.append(p)
Object = Basis('Total order', listing)
#Establish n Parameter objects
params = []
P = Parameter(order=p, lower=-1, upper=1, distribution='uniform')
for i in range(0, n):
params.append(P)
#Use the params list to establish the Poly object
Polybasis = Poly(params, Object)
V = Polybasis.getPolynomial(eta)
V = V.T
return V, Polybasis
def get_subspace_polynomial(self):
""" Returns a polynomial defined over the dimension reducing subspace.
Returns
-------
Poly
A Poly object that defines a polynomial over the subspace. The distribution of parameters
is assumed to be uniform and the maximum and minimum bounds for each parameter are defined by the maximum
and minimum values of the project samples.
"""
# TODO: Try correlated poly here
active_subspace = self._subspace[:, 0:self.subspace_dimension]
projected_points = np.dot(self.std_sample_points, active_subspace)
myparameters = []
for i in range(0, self.subspace_dimension):
param = Parameter(distribution='uniform',
lower=np.min(projected_points[:, i]),
upper=np.max(projected_points[:, i]),
order=self.polynomial_degree)
myparameters.append(param)
mybasis = Basis("total-order")
subspacepoly = Poly(myparameters,
mybasis,
method='least-squares',
sampling_args={
'sample-points': projected_points,
'sample-outputs': self.sample_outputs
})
subspacepoly.set_model()
return subspacepoly
def __init__(self, method, full_space_poly=None, sample_points=None, sample_outputs=None, polynomial_degree=2, subspace_dimension=2, bootstrap=False, subspace_init=None, max_iter=1000, tol=None, poly_method='least-squares',solver_args=None):
self.full_space_poly = full_space_poly
self.sample_points = sample_points
self.Y = None # for the zonotope vertices
if self.sample_points is not None:
self.sample_points = standardise(sample_points)
self.sample_outputs = sample_outputs
self.method = method
self.subspace_dimension = subspace_dimension
self.polynomial_degree = polynomial_degree
self.bootstrap = bootstrap
self.poly_method = poly_method
self.solver_args = solver_args
if self.method.lower() == 'active-subspace' or self.method.lower() == 'active-subspaces':
self.method = 'active-subspace'
if self.full_space_poly is None:
N, d = self.sample_points.shape
param = Parameter(distribution='uniform', lower=-1, upper=1., order=self.polynomial_degree)
myparameters = [param for _ in range(d)]
mybasis = Basis("total-order")
mypoly = Poly(myparameters, mybasis, method=self.poly_method, sampling_args={'sample-points':self.sample_points, \
'sample-outputs':self.sample_outputs},
solver_args=self.solver_args)
mypoly.set_model()
self.full_space_poly = mypoly
self.sample_points = standardise(self.full_space_poly.get_points())
self.sample_outputs = self.full_space_poly.get_model_evaluations()
self._get_active_subspace()
elif self.method == 'variable-projection':
self._get_variable_projection(None,None,tol,max_iter,subspace_init,False)
def _get_quadrature_points_and_weights(self, order):
param = Parameter(distribution='uniform',
lower=self.lower,
upper=self.upper,
order=order)
basis = Basis('univariate')
poly = Poly(method='numerical-integration',
parameters=param,
basis=basis)
points, weights = poly.get_points_and_weights()
return points, weights * (self.upper - self.lower)
def vandermonde(eta, p):
_, n = eta.shape
listing = []
for i in range(0, n):
listing.append(p)
Object = Basis('total-order', listing)
#Establish n Parameter objects
params = []
P = Parameter(order=p, lower=-1, upper=1, distribution='uniform')
for i in range(0, n):
params.append(P)
#Use the params list to establish the Poly object
Polybasis = Poly(params, Object, method='least-squares')
V = Polybasis.get_poly(eta)
V = V.T
return V, Polybasis
def __init__(self, training_input, training_output, num_ridges, max_iters=1, learning_rate = 0.001,
W=None, coeffs=None, momentum_rate = .001, opt = 'sd', poly_deg = 2, verbose = False):
self.training_input = training_input
self.training_output = training_output
self.verbose = verbose
# network architecture params
if isinstance(num_ridges, int):
self.num_ridges = [num_ridges]
else:
self.num_ridges = num_ridges
# num_ridges is the number of hidden units at each hidden layer. Does not count the input layer
self.num_layers = len(self.num_ridges)
self.dims = training_input.shape[1]
# initialize network data structures
max_layer_size = max(self.num_ridges)
self.poly_array = np.empty(( self.num_layers, max_layer_size), dtype=object)
#TODO: not hardcode poly type? Have different ridges at different nodes?
for k in range(self.num_layers):
for j in range(self.num_ridges[k]):
self.poly_array[k,j] = Poly(Parameter(poly_deg, distribution='uniform', lower=-3, upper=3), Basis("total order"))
self.poly_card = self.poly_array[0,0].basis.cardinality
layer_sizes = [self.dims] + self.num_ridges
if W is None:
self.W = [np.random.randn(layer_sizes[k+1], layer_sizes[k]) for k in range(self.num_layers)]
else:
self.W = W
if coeffs is None:
self.coeffs = [np.random.randn(self.num_ridges[k], self.poly_card) for k in range(self.num_layers)]
else:
self.coeffs = coeffs
self.update_coeffs()
# Note: We will keep data for every input point in one array.
n_points = self.training_input.shape[0]
self.delta = []
for k in range(self.num_layers):
self.delta.append(np.zeros((self.num_ridges[k],n_points)))
self.act_mat = [] # Lambda
for k in range(self.num_layers):
self.act_mat.append(np.zeros((self.num_ridges[k], n_points)))
self.Z = [] # node value before activation
for k in range(self.num_layers):
self.Z.append(np.zeros((self.num_ridges[k],n_points)))
self.Y = [] # After activation
for k in range(self.num_layers):
self.Y.append(np.zeros((self.num_ridges[k],n_points)))
self.phi = [] # basis fn evaluations
for k in range(self.num_layers):
self.phi.append(np.zeros((self.num_ridges[k],n_points)))
self.evaluate_fit(self.training_input,train=True)
# optimization params
self.max_iters = max_iters
self.opt = opt
self.learning_rate = learning_rate
self.momentum_rate = momentum_rate
def _calculate_subspace(self, S, f):
parameters = [Parameter(distribution='uniform', lower=np.min(S[:,i]), upper=np.max(S[:,i]), order=1) for i in range(0, self.n)]
self.poly = Poly(parameters, basis=Basis('total-order'), method='least-squares', \
sampling_args={'sample-points': S, 'sample-outputs': f})
self.poly.set_model()
self.Subs = Subspaces(full_space_poly=self.poly, method='active-subspace', subspace_dimension=self.d)
if self.subspace_method == 'variable-projection':
U0 = self.Subs.get_subspace()[:,:self.d]
self.Subs = Subspaces(method='variable-projection', sample_points=S, sample_outputs=f, \
subspace_init=U0, subspace_dimension=self.d, polynomial_degree=2, max_iter=300, tol=1.0e-8)
self.U = self.Subs.get_subspace()[:, :self.d]
elif self.subspace_method == 'active-subspaces':
U0 = self.Subs.get_subspace()[:,1].reshape(-1,1)
U1 = null_space(U0.T)
self.U = U0
for i in range(self.d-1):
R = []
for j in range(U1.shape[1]):
U = np.hstack((self.U, U1[:, j].reshape(-1,1)))
Y = np.dot(S, U)
myParameters = [Parameter(distribution='uniform', lower=np.min(Y[:,k]), upper=np.max(Y[:,k]), \
order=2) for k in range(Y.shape[1])]
myBasis = Basis('total-order')
poly = Poly(myParameters, myBasis, method='least-squares', \
sampling_args={'sample-points':Y, 'sample-outputs':f})
poly.set_model()
f_eval = poly.get_polyfit(Y)
_,_,r,_,_ = linregress(f_eval.flatten(),f.flatten())
R.append(r**2)
index = np.argmax(R)
self.U = np.hstack((self.U, U1[:, index].reshape(-1,1)))
U1 = np.delete(U1, index, 1)
def _build_model(self, S, f, del_k):
"""
Constructs quadratic model for ``trust-region`` method
"""
myParameters = [
Parameter(distribution='uniform',
lower=S[0, i] - del_k,
upper=S[0, i] + del_k,
order=2) for i in range(S.shape[1])
]
myBasis = Basis('total-order')
my_poly = Poly(myParameters,
myBasis,
method='compressive-sensing',
sampling_args={
'sample-points': S,
'sample-outputs': f
})
my_poly.set_model()
return my_poly
def test_sampling(self):
d = 4
order = 5
param = Parameter(distribution='uniform',
order=order,
lower=-1.0, upper=1.0)
myparameters = [param for _ in range(d)]
mybasis = Basis('total-order')
mypoly = Poly(myparameters, mybasis,
method='least-squares',
sampling_args={'mesh': 'induced',
'subsampling-algorithm': 'qr',
'sampling-ratio': 1})
assert mypoly._quadrature_points.shape == (mypoly.basis.cardinality, d)
def _fit_poly(X, y):
N, d = X.shape
myParameters = []
for dimension in range(d):
values = [X[i,dimension] for i in range(N)]
values_min = min(values)
values_max = max(values)
if (values_min - values_max) ** 2 < 0.01:
myParameters.append(Parameter(distribution='Uniform', lower=values_min-0.01, upper=values_max+0.01, order=self.order))
else:
myParameters.append(Parameter(distribution='Uniform', lower=values_min, upper=values_max, order=self.order))
myBasis = Basis('total-order')
y = np.reshape(y, (y.shape[0], 1))
poly = Poly(myParameters, myBasis, method='least-squares', sampling_args={'sample-points':X, 'sample-outputs':y})
poly.set_model()
mse = ((y-poly.get_polyfit(X))**2).mean()
return mse, poly
def _build_model(self, S, f):
"""
Constructs quadratic model for ``trust-region`` or ``omorf`` methods
"""
if self.method == 'trust-region':
myParameters = [Parameter(distribution='uniform', lower=np.min(S[:,i]), \
upper=np.max(S[:,i]), order=2) for i in range(self.n)]
myBasis = Basis('total-order')
my_poly = Poly(myParameters, myBasis, method='least-squares', \
sampling_args={'sample-points':S, 'sample-outputs':f})
elif self.method == 'omorf':
Y = np.dot(S, self.U)
myParameters = [Parameter(distribution='uniform', lower=np.min(Y[:,i]), \
upper=np.max(Y[:,i]), order=2) for i in range(self.d)]
myBasis = Basis('total-order')
my_poly = Poly(myParameters, myBasis, method='least-squares', \
sampling_args={'sample-points':Y, 'sample-outputs':f})
my_poly.set_model()
return my_poly
def __init__(self, poly, correlation_matrix, verbose=False):
self.poly = poly
D = self.poly.get_parameters()
self.D = D
self.R = correlation_matrix
self.std = Parameter(order=5,
distribution='normal',
shape_parameter_A=0.0,
shape_parameter_B=1.0)
inf_lim = -8.0
sup_lim = -inf_lim
p1 = Parameter(distribution='uniform',
lower=inf_lim,
upper=sup_lim,
order=31)
myBasis = Basis('tensor-grid')
Pols = Poly([p1, p1], myBasis, method='numerical-integration')
p = Pols.get_points()
w = Pols.get_weights() * (sup_lim - inf_lim)**2
p1 = p[:, 0]
p2 = p[:, 1]
R0 = np.eye((len(self.D)))
for i in range(len(self.D)):
for j in range(i + 1, len(self.D), 1):
if self.R[i, j] == 0:
R0[i, j] = 0.0
else:
tp11 = -(np.array(self.D[i].get_icdf(
self.std.get_cdf(points=p1))) -
self.D[i].mean) / np.sqrt(self.D[i].variance)
tp22 = -(np.array(self.D[j].get_icdf(
self.std.get_cdf(points=p2))) -
self.D[j].mean) / np.sqrt(self.D[j].variance)
rho_ij = self.R[i, j]
bivariateNormalPDF = (
1.0 / (2.0 * np.pi * np.sqrt(1.0 - rho_ij**2)) *
np.exp(-1.0 / (2.0 * (1.0 - rho_ij**2)) *
(p1**2 - 2.0 * rho_ij * p1 * p2 + p2**2)))
coefficientsIntegral = np.flipud(tp11 * tp22 * w)
def check_difference(rho_ij):
bivariateNormalPDF = (
1.0 / (2.0 * np.pi * np.sqrt(1.0 - rho_ij**2)) *
np.exp(-1.0 / (2.0 * (1.0 - rho_ij**2)) *
(p1**2 - 2.0 * rho_ij * p1 * p2 + p2**2)))
diff = np.dot(coefficientsIntegral, bivariateNormalPDF)
return diff - self.R[i, j]
if (self.D[i].name != 'custom') or (self.D[j].name !=
'custom'):
rho = optimize.newton(check_difference,
self.R[i, j],
maxiter=50)
else:
res = optimize.least_squares(check_difference,
R[i, j],
bounds=(-0.999, 0.999),
ftol=1.e-03)
rho = res.x
print('A Custom Marginal is present')
R0[i, j] = rho
R0[j, i] = R0[i, j]
self.A = np.linalg.cholesky(R0)
if verbose is True:
print('The Cholesky decomposition of fictive matrix R0 is:')
print(self.A)
print('The fictive matrix is:')
print(R0)
list_of_parameters = []
for i in range(0, len(self.D)):
standard_parameter = Parameter(order=self.D[i].order,
distribution='gaussian',
shape_parameter_A=0.,
shape_parameter_B=1.)
list_of_parameters.append(standard_parameter)
self.polystandard = deepcopy(self.poly)
self.polystandard._set_parameters(list_of_parameters)
self.standard_samples = self.polystandard.get_points()
self._points = self.get_correlated_from_uncorrelated(
self.standard_samples)
def approxFullSpacePolynomial(self):
"""
Use the quadratic program to approximate the polynomial over the full space.
"""
Polyfull = Poly()
return Polyfull
def __init__(self,
correlation_matrix,
poly=None,
parameters=None,
method=None,
verbose=False):
if (poly is None) and (method is not None):
raise ValueError('Need to specify poly for probability transform.')
if poly is not None:
self.poly = poly
D = self.poly.get_parameters()
elif parameters is not None:
D = parameters
else:
raise ValueError('Need to specify either poly or parameters.')
self.D = D
self.R = correlation_matrix
self.std = Parameter(order=5,
distribution='normal',
shape_parameter_A=0.0,
shape_parameter_B=1.0)
inf_lim = -8.0
sup_lim = -inf_lim
p1 = Parameter(distribution='uniform',
lower=inf_lim,
upper=sup_lim,
order=31)
myBasis = Basis('tensor-grid')
self.Pols = Poly([p1, p1], myBasis, method='numerical-integration')
Pols = self.Pols
p = Pols.get_points()
# w = Pols.get_weights()
w = Pols.get_weights() * (sup_lim - inf_lim)**2
p1 = p[:, 0]
p2 = p[:, 1]
R0 = np.eye((len(self.D)))
for i in range(len(self.D)):
for j in range(i + 1, len(self.D), 1):
if self.R[i, j] == 0:
R0[i, j] = 0.0
else:
z1 = np.array(self.D[i].get_icdf(
self.std.get_cdf(points=p1)))
z2 = np.array(self.D[j].get_icdf(
self.std.get_cdf(points=p2)))
tp11 = (z1 - self.D[i].mean) / np.sqrt(self.D[i].variance)
tp22 = (z2 - self.D[j].mean) / np.sqrt(self.D[j].variance)
coefficientsIntegral = np.flipud(tp11 * tp22 * w)
def check_difference(rho_ij):
bivariateNormalPDF = (
1.0 / (2.0 * np.pi * np.sqrt(1.0 - rho_ij**2)) *
np.exp(-1.0 / (2.0 * (1.0 - rho_ij**2)) *
(p1**2 - 2.0 * rho_ij * p1 * p2 + p2**2)))
diff = np.dot(coefficientsIntegral, bivariateNormalPDF)
return diff - self.R[i, j]
# if (self.D[i].name!='custom') or (self.D[j].name!='custom'):
rho = optimize.newton(check_difference,
self.R[i, j],
maxiter=50)
# else:
# # ???
# res = optimize.least_squares(check_difference, self.R[i,j], bounds=(-0.999,0.999), ftol=1.e-03)
# rho = res.x
# print('A Custom Marginal is present')
R0[i, j] = rho
R0[j, i] = R0[i, j]
self.R0 = R0.copy()
self.A = np.linalg.cholesky(R0)
if verbose:
print('The Cholesky decomposition of fictive matrix R0 is:')
print(self.A)
print('The fictive matrix is:')
print(R0)
if method is None:
pass
elif method.lower() == 'nataf-transform':
list_of_parameters = []
for i in range(0, len(self.D)):
standard_parameter = Parameter(order=self.D[i].order,
distribution='gaussian',
shape_parameter_A=0.,
shape_parameter_B=1.)
list_of_parameters.append(standard_parameter)
# have option so that we don't need to obtain
self.corrected_poly = deepcopy(self.poly)
if hasattr(self.corrected_poly, '_quadrature_points'):
self.corrected_poly._set_parameters(list_of_parameters)
self.standard_samples = self.corrected_poly._quadrature_points
self._points = self.get_correlated_samples(
X=self.standard_samples)
# self.corrected_poly._quadrature_points = self._points.copy()
elif method.lower() == 'gram-schmidt':
basis_card = poly.basis.cardinality
oversampling = 10
N_Psi = oversampling * basis_card
S_samples = self.get_correlated_samples(N=N_Psi)
w_weights = 1.0 / N_Psi * np.ones(N_Psi)
Psi = poly.get_poly(S_samples).T
WPsi = np.diag(np.sqrt(w_weights)) @ Psi
self.WPsi = WPsi
R_Psi = np.linalg.qr(WPsi)[1]
self.R_Psi = R_Psi
self.R_Psi[0, :] *= np.sign(self.R_Psi[0, 0])
self.corrected_poly = deepcopy(poly)
self.corrected_poly.inv_R_Psi = np.linalg.inv(self.R_Psi)
self.corrected_poly.corr = self
self.corrected_poly._set_points_and_weights()
P = self.corrected_poly.get_poly(
self.corrected_poly._quadrature_points)
W = np.mat(
np.diag(np.sqrt(self.corrected_poly._quadrature_weights)))
A = W * P.T
self.corrected_poly.A = A
self.corrected_poly.P = P
if hasattr(self.corrected_poly, '_quadrature_points'):
# TODO: Correlated quadrature points?
self._points = self.corrected_poly._quadrature_points
else:
raise ValueError('Invalid method for correlations.')
def __init__(self,
method,
full_space_poly=None,
sample_points=None,
sample_outputs=None,
subspace_dimension=2,
polynomial_degree=2,
param_args=None,
poly_args=None,
dr_args=None):
self.full_space_poly = full_space_poly
self.sample_points = sample_points
self.Y = None # for the zonotope vertices
self.sample_outputs = sample_outputs
self.method = method
self.subspace_dimension = subspace_dimension
self.polynomial_degree = polynomial_degree
my_poly_args = {'method': 'least-squares', 'solver_args': {}}
if poly_args is not None:
my_poly_args.update(poly_args)
self.poly_args = my_poly_args
my_param_args = {
'distribution': 'uniform',
'order': self.polynomial_degree,
'lower': -1,
'upper': 1
}
if param_args is not None:
my_param_args.update(param_args)
# I suppose we can detect if lower and upper is present to decide between these categories?
bounded_distrs = [
'analytical', 'beta', 'chebyshev', 'arcsine', 'truncated-gaussian',
'uniform'
]
unbounded_distrs = [
'gaussian', 'normal', 'gumbel', 'logistic', 'students-t',
'studentst'
]
semi_bounded_distrs = [
'chi', 'chi-squared', 'exponential', 'gamma', 'lognormal',
'log-normal', 'pareto', 'rayleigh', 'weibull'
]
if dr_args is not None:
if 'standardize' in dr_args:
dr_args['standardise'] = dr_args['standardize']
if self.method.lower() == 'active-subspace' or self.method.lower(
) == 'active-subspaces':
self.method = 'active-subspace'
if dr_args is not None:
self.standardise = getattr(dr_args, 'standardise', True)
else:
self.standardise = True
if self.full_space_poly is None:
# user provided input/output data
N, d = self.sample_points.shape
if self.standardise:
self.data_scaler = scaler_minmax()
self.data_scaler.fit(self.sample_points)
self.std_sample_points = self.data_scaler.transform(
self.sample_points)
else:
self.std_sample_points = self.sample_points.copy()
param = Parameter(**my_param_args)
if param_args is not None:
if (hasattr(dr_args, 'lower')
or hasattr(dr_args, 'upper')) and self.standardise:
warnings.warn(
'Points standardised but parameter range provided. Overriding default ([-1,1])...',
UserWarning)
myparameters = [param for _ in range(d)]
mybasis = Basis("total-order")
mypoly = Poly(myparameters,
mybasis,
sampling_args={
'sample-points': self.std_sample_points,
'sample-outputs': self.sample_outputs
},
**my_poly_args)
mypoly.set_model()
self.full_space_poly = mypoly
else:
# User provided polynomial
# Standardise according to distribution specified. Only care about the scaling (not shift)
# TODO: user provided callable with parameters?
user_params = self.full_space_poly.parameters
d = len(user_params)
self.sample_points = self.full_space_poly.get_points()
if self.standardise:
scale_factors = np.zeros(d)
centers = np.zeros(d)
for dd, p in enumerate(user_params):
if p.name.lower() in bounded_distrs:
scale_factors[dd] = (p.upper - p.lower) / 2.0
centers[dd] = (p.upper + p.lower) / 2.0
elif p.name.lower() in unbounded_distrs:
scale_factors[dd] = np.sqrt(p.variance)
centers[dd] = p.mean
else:
scale_factors[dd] = np.sqrt(p.variance)
centers[dd] = 0.0
self.param_scaler = scaler_custom(centers, scale_factors)
self.std_sample_points = self.param_scaler.transform(
self.sample_points)
else:
self.std_sample_points = self.sample_points.copy()
if not hasattr(self.full_space_poly, 'coefficients'):
raise ValueError('Please call set_model() first on poly.')
self.sample_outputs = self.full_space_poly.get_model_evaluations()
# TODO: use dr_args for resampling of gradient points
as_args = {'grad_points': None}
if dr_args is not None:
as_args.update(dr_args)
self._get_active_subspace(**as_args)
elif self.method == 'variable-projection':
self.data_scaler = scaler_minmax()
self.data_scaler.fit(self.sample_points)
self.std_sample_points = self.data_scaler.transform(
self.sample_points)
if dr_args is not None:
vp_args = {
'gamma': 0.1,
'beta': 1e-4,
'tol': 1e-7,
'maxiter': 1000,
'U0': None,
'verbose': False
}
vp_args.update(dr_args)
self._get_variable_projection(**vp_args)
else:
self._get_variable_projection()
class Optimisation:
"""
This class performs unconstrained or constrained optimisation of poly objects or custom functions
using scipy.optimize.minimize or an in-house trust-region method.
:param string method: A string specifying the method that will be used for optimisation. All of the available choices come from scipy.optimize.minimize (`click here <https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.minimize.html>`__ for a list of methods and further information). In the case of general constrained optimisation, the options are ``COBYLA``, ``SLSQP``, and ``trust-constr``. The default is ``trust-constr``.
"""
def __init__(self, method):
self.method = method
self.objective = {'function': None, 'gradient': None, 'hessian': None}
self.maximise = False
self.bounds = None
self.constraints = []
self.num_evals = 0
# np.random.seed(42)
if self.method in ['trust-region', 'omorf']:
self.num_evals = 0
self.S = np.array([])
self.f = np.array([])
self.g = np.array([])
def add_objective(self, poly=None, custom=None, maximise=False):
"""
Adds objective function to be optimised.
:param poly poly:
A Poly object.
:param dict custom: Optional arguments centered around the custom option.
:callable function: The objective function to be called.
:callable jac_function: The gradient (or derivative) of the objective.
:callable hess_function: The Hessian of the objective function.
:param bool maximise: A flag to specify if the user would like to maximise the function instead of minimising it.
"""
assert poly is not None or custom is not None
if self.method == 'trust-region':
assert poly is None
assert custom is not None
self.maximise = maximise
k = 1.0
if self.maximise:
k = -1.0
if poly is not None:
f = poly.get_polyfit_function()
jac = poly.get_polyfit_grad_function()
hess = poly.get_polyfit_hess_function()
objective = lambda x: k * np.asscalar(f(x))
objective_deriv = lambda x: k * jac(x)[:, 0]
objective_hess = lambda x: k * hess(x)[:, :, 0]
elif custom is not None:
assert 'function' in custom
objective = lambda s: k * custom['function'](s)
if 'jac_function' in custom:
objective_deriv = lambda s: k * custom['jac_function'](s)
else:
objective_deriv = '2-point'
if 'hess_function' in custom:
objective_hess = lambda s: k * custom['hess_function'](s)
else:
objective_hess = optimize.BFGS()
self.objective = {
'function': objective,
'gradient': objective_deriv,
'hessian': objective_hess
}
def add_bounds(self, lb, ub):
"""
Adds bounds :math:`lb <= x <=ub` to the optimisation problem. Only ``L-BFGS-B``, ``TNC``, ``SLSQP``, ``trust-constr``, ``trust-region``, and ``COBYLA`` methods can handle bounds.
:param numpy.ndarray lb: 1-by-n matrix that contains lower bounds of x.
:param numpy.ndarray ub: 1-by-n matrix that contains upper bounds of x.
"""
assert lb.size == ub.size
assert self.method in [
'L-BFGS-B', 'TNC', 'SLSQP', 'trust-constr', 'COBYLA',
'trust-region', 'omorf'
]
if self.method in ['trust-region', 'omorf']:
self.bounds = [lb, ub]
elif self.method != 'COBYLA':
self.bounds = []
for i in range(lb.size):
self.bounds.append((lb[i], ub[i]))
self.bounds = tuple(self.bounds)
else:
for factor in range(lb.size):
if not np.isinf(lb[factor]):
l = {
'type': 'ineq',
'fun': lambda x, i=factor: x[i] - lb[i]
}
self.constraints.append(l)
if not np.isinf(ub[factor]):
u = {
'type': 'ineq',
'fun': lambda x, i=factor: ub[i] - x[i]
}
self.constraints.append(u)
def add_linear_ineq_con(self, A, b_l, b_u):
"""
Adds linear inequality constraints :math:`b_l <= A x <= b_u` to the optimisation problem.
Only ``trust-constr``, ``COBYLA``, and ``SLSQP`` methods can handle general constraints.
:param numpy.ndarray A: An (M,n) matrix that contains coefficients of the linear inequality constraints.
:param numpy.ndarray b_l: An (M,1) matrix that specifies lower bounds of the linear inequality constraints. If there is no lower bound, set ``b_l = -np.inf * np.ones(M)``.
:param numpy.ndarray b_u: A (M,1) matrix that specifies upper bounds of the linear inequality constraints. If there is no upper bound, set ``b_u = np.inf * np.ones(M)``.
"""
# trust-constr method has its own linear constraint handler
assert self.method in ['SLSQP', 'trust-constr', 'COBYLA']
if self.method == 'trust-constr':
self.constraints.append(optimize.LinearConstraint(A, b_l, b_u))
# other methods add inequality constraints using dictionary files
else:
if not np.any(np.isinf(b_l)):
self.constraints.append({
'type': 'ineq',
'fun': lambda x: np.dot(A, x) - b_l,
'jac': lambda x: A
})
if not np.any(np.isinf(b_u)):
self.constraints.append({
'type': 'ineq',
'fun': lambda x: -np.dot(A, x) + b_u,
'jac': lambda x: -A
})
def add_nonlinear_ineq_con(self, poly=None, custom=None):
"""
Adds nonlinear inequality constraints :math:`lb <= g(x) <= ub` (for poly option) with :math:`lb`, :math:`ub = bounds` or :math:`g(x) >= 0` (for function option) to the optimisation problem. Only ``trust-constr``, ``COBYLA``, and ``SLSQP`` methods can handle general constraints.
If Poly object is provided in the poly dictionary, gradients and Hessians will be computed automatically. If a lambda function is provided in the ``function`` dictionary, the user may also provide ``jac_function`` for gradients and ``hess_function`` for Hessians; otherwise, a 2-point differentiation rule
will be used to approximate the derivative and a BFGS update will be used to approximate the Hessian.
:param dict poly: Arguments for poly dictionary.
:param Poly poly: An instance of the Poly class.
:param numpy.ndarray bounds: An array with two entries specifying the lower and upper bounds of the inequality. If there is no lower bound, set bounds[0] = -np.inf.If there is no upper bound, set bounds[1] = np.inf.
:param dict custom: Additional custom callable arguments.
:callable function: The constraint function to be called.
:callable jac_function: The gradient (or derivative) of the constraint.
:callable hess_function: The Hessian of the constraint function.
"""
assert self.method in ['SLSQP', 'trust-constr', 'COBYLA']
assert poly is not None or custom is not None
if poly is not None:
assert 'bounds' in poly
bounds = poly['bounds']
assert 'poly' in poly
gpoly = poly['poly']
# Get lambda functions for function, gradient, and Hessians from poly object
g = gpoly.get_polyfit_function()
jac = gpoly.get_polyfit_grad_function()
hess = gpoly.get_polyfit_hess_function()
constraint = lambda x: g(x)[0]
constraint_deriv = lambda x: jac(x)[:, 0]
constraint_hess = lambda x, v: hess(x)[:, :, 0]
if self.method == 'trust-constr':
self.constraints.append(optimize.NonlinearConstraint(constraint, bounds[0], bounds[1], \
jac = constraint_deriv, hess = constraint_hess))
# other methods add inequality constraints using dictionary files
elif self.method == 'SLSQP':
if not np.isinf(bounds[0]):
self.constraints.append({'type':'ineq', 'fun': lambda x: constraint(x) - bounds[0], \
'jac': constraint_deriv})
if not np.isinf(bounds[1]):
self.constraints.append({'type':'ineq', 'fun': lambda x: -constraint(x) + bounds[1], \
'jac': lambda x: -constraint_deriv(x)})
else:
if not np.isinf(bounds[0]):
self.constraints.append({
'type':
'ineq',
'fun':
lambda x: constraint(x) - bounds[0]
})
if not np.isinf(bounds[1]):
self.constraints.append({
'type':
'ineq',
'fun':
lambda x: -constraint(x) + bounds[1]
})
elif custom is not None:
assert 'function' in custom
constraint = custom['function']
if 'jac_function' in custom:
constraint_deriv = custom['jac_function']
else:
constraint_deriv = '2-point'
if 'hess_function' in custom:
constraint_hess = lambda x, v: custom['hess_function'](x)
else:
constraint_hess = optimize.BFGS()
if self.method == 'trust-constr':
self.constraints.append(optimize.NonlinearConstraint(constraint, 0.0, np.inf, jac = constraint_deriv, \
hess = constraint_hess))
elif self.method == 'SLSQP':
if 'jac_function' in custom:
self.constraints.append({
'type': 'ineq',
'fun': constraint,
'jac': constraint_deriv
})
else:
self.constraints.append({
'type': 'ineq',
'fun': constraint
})
else:
self.constraints.append({'type': 'ineq', 'fun': constraint})
def add_linear_eq_con(self, A, b):
"""
Adds linear equality constraints :math:`Ax = b` to the optimisation routine. Only ``trust-constr`` and ``SLSQP`` methods can handle equality constraints.
:param numpy.ndarray A: A (M, n) matrix that contains coefficients of the linear equality constraints.
:param numpy.ndarray b: A (M, 1) matrix that specifies right hand side of the linear equality constraints.
"""
assert self.method == 'trust-constr' or 'SLSQP'
if self.method == 'trust-constr':
self.constraints.append(optimize.LinearConstraint(A, b, b))
else:
self.constraints.append({
'type': 'eq',
'fun': lambda x: A.dot(x) - b,
'jac': lambda x: A
})
def add_nonlinear_eq_con(self, poly=None, custom=None):
"""
Adds nonlinear inequality constraints :math:`g(x) = value` (for poly option) or :math:`g(x) = 0` (for function option) to the optimisation routine.
Only ``trust-constr`` and ``SLSQP`` methods can handle equality constraints. If poly object is providedin the poly dictionary, gradients and Hessians will be computed automatically.
:param dict poly: Arguments for poly dictionary.
:param Poly poly: An instance of the Poly class.
:param float value: Value of the nonlinear constraint.
:param dict custom: Additional custom callable arguments.
:callable function: The constraint function to be called.
:callable jac_function: The gradient (or derivative) of the constraint.
:callable hess_function: The Hessian of the constraint function.
"""
assert self.method == 'trust-constr' or 'SLSQP'
assert poly is not None or custom is not None
if poly is not None:
assert 'value' in poly
value = poly['value']
g = poly.get_polyfit_function()
jac = poly.get_polyfit_grad_function()
hess = poly.get_polyfit_hess_function()
constraint = lambda x: np.asscalar(g(x))
constraint_deriv = lambda x: jac(x)[:, 0]
constraint_hess = lambda x, v: hess(x)[:, :, 0]
if self.method == 'trust-constr':
self.constraints.append(optimize.NonlinearConstraint(constraint, value, value, jac=constraint_deriv, \
hess=constraint_hess))
else:
self.constraints.append({'type':'eq', 'fun': lambda x: constraint(x) - value, \
'jac': constraint_deriv})
elif custom is not None:
assert 'function' in custom
constraint = custom['function']
if 'jac_function' in custom:
constraint_deriv = custom['jac_function']
else:
constraint_deriv = '2-point'
if 'hess_function' in custom:
constraint_hess = lambda x, v: custom['hess_function'](x)
else:
constraint_hess = optimize.BFGS()
if self.method == 'trust-constr':
self.constraints.append(optimize.NonlinearConstraint(constraint, 0.0, 0.0, jac=constraint_deriv, \
hess=constraint_hess))
else:
if 'jac_function' in custom:
self.constraints.append({
'type': 'eq',
'fun': constraint,
'jac': constraint_deriv
})
else:
self.constraints.append({'type': 'eq', 'fun': constraint})
def optimise(self, x0, *args, **kwargs):
"""
Performs optimisation on a specified function, provided the objective has been added using 'add_objective' method
and constraints have been added using the relevant method.
:param numpy.ndarray x0: Starting point for optimiser.
:param float del_k: initial trust-region radius for ``trust-region`` or ``omorf`` methods
:param float delmin: minimum allowable trust-region radius for ``trust-region`` or ``omorf`` methods
:param float delmax: maximum allowable trust-region radius for ``trust-region`` or ``omorf`` methods
:param int d: reduced dimension for ``omorf`` method
:param string subspace_method: subspace method for ``omorf`` method with options ``variable-projection`` or ``active-subspaces``
:return:
**sol**: An object containing the optimisation result. Important attributes are: the solution array ``x``, and a Boolean flag ``success`` indicating
if the optimiser exited successfully.
"""
assert self.objective['function'] is not None
if self.method in [
'Newton-CG', 'dogleg', 'trust-ncg', 'trust-krylov',
'trust-exact', 'trust-constr'
]:
sol = optimize.minimize(self.objective['function'], x0, method=self.method, bounds = self.bounds, \
jac=self.objective['gradient'], hess=self.objective['hessian'], \
constraints=self.constraints, options={'disp': False, 'maxiter': 10000})
sol = {
'x': sol['x'],
'fun': sol['fun'],
'nfev': self.num_evals,
'status': sol['status']
}
elif self.method in ['CG', 'BFGS', 'L-BFGS-B', 'TNC', 'SLSQP']:
sol = optimize.minimize(self.objective['function'], x0, method=self.method, bounds = self.bounds, \
jac=self.objective['gradient'], constraints=self.constraints, \
options={'disp': False, 'maxiter': 10000})
sol = {
'x': sol['x'],
'fun': sol['fun'],
'nfev': self.num_evals,
'status': sol['status']
}
elif self.method in ['trust-region']:
x_opt, f_opt = self._trust_region(x0, del_k=kwargs.get('del_k', None), del_min=kwargs.get('delmin', 1.0e-8), \
eta1=kwargs.get('eta1', 0.0), eta2=kwargs.get('eta2', 0.7), gam1=kwargs.get('gam1', 0.5), \
gam2=kwargs.get('gam2', 2.0), omega_s=kwargs.get('omega_s', 0.5), max_evals=kwargs.get('max_evals', 10000), \
random_initial=kwargs.get('random_initial', False), epsilon=kwargs.get('epsilon', 1.05))
sol = {'x': x_opt, 'fun': f_opt, 'nfev': self.num_evals}
elif self.method in ['omorf']:
x_opt, f_opt = self._omorf(x0, del_k=kwargs.get('del_k', None), del_min=kwargs.get('delmin', 1.0e-8), \
eta1=kwargs.get('eta1', 0.1), eta2=kwargs.get('eta2', 0.7), gam1=kwargs.get('gam1', 0.5), \
gam2=kwargs.get('gam2', 2.0), omega_s=kwargs.get('omega_s', 0.5), max_evals=kwargs.get('max_evals', 10000), \
random_initial=kwargs.get('random_initial', False), epsilon=kwargs.get('epsilon', 1.05), \
d=kwargs.get('d', 2), subspace_method=kwargs.get('subspace_method', 'variable-projection'))
sol = {'x': x_opt, 'fun': f_opt, 'nfev': self.num_evals}
else:
sol = optimize.minimize(self.objective['function'], x0, method=self.method, bounds = self.bounds, \
constraints=self.constraints, options={'disp': False, 'maxiter': 10000})
sol = {
'x': sol['x'],
'fun': sol['fun'],
'nfev': self.num_evals,
'status': sol['status']
}
if self.maximise:
sol['fun'] *= -1.0
return sol
def _calculate_subspace(self, S, f):
parameters = [
Parameter(distribution='uniform',
lower=np.min(S[:, i]),
upper=np.max(S[:, i]),
order=1) for i in range(0, self.n)
]
self.poly = Poly(parameters, basis=Basis('total-order'), method='least-squares', \
sampling_args={'sample-points': S, 'sample-outputs': f})
self.poly.set_model()
self.Subs = Subspaces(full_space_poly=self.poly,
method='active-subspace',
subspace_dimension=self.d)
if self.subspace_method == 'variable-projection':
U0 = self.Subs.get_subspace()[:, :self.d]
self.Subs = Subspaces(method='variable-projection', sample_points=S, sample_outputs=f, \
subspace_init=U0, subspace_dimension=self.d, polynomial_degree=2, max_iter=300)
self.U = self.Subs.get_subspace()[:, :self.d]
elif self.subspace_method == 'active-subspaces':
U0 = self.Subs.get_subspace()[:, 1].reshape(-1, 1)
U1 = null_space(U0.T)
self.U = U0
for i in range(self.d - 1):
R = []
for j in range(U1.shape[1]):
U = np.hstack((self.U, U1[:, j].reshape(-1, 1)))
Y = np.dot(S, U)
myParameters = [Parameter(distribution='uniform', lower=np.min(Y[:,k]), upper=np.max(Y[:,k]), \
order=2) for k in range(Y.shape[1])]
myBasis = Basis('total-order')
poly = Poly(myParameters, myBasis, method='least-squares', \
sampling_args={'sample-points':Y, 'sample-outputs':f})
poly.set_model()
f_eval = poly.get_polyfit(Y)
_, _, r, _, _ = linregress(f_eval.flatten(), f.flatten())
R.append(r**2)
index = np.argmax(R)
self.U = np.hstack((self.U, U1[:, index].reshape(-1, 1)))
U1 = np.delete(U1, index, 1)
def _blackbox_evaluation(self, s):
"""
Evaluates the point s for ``trust-region`` or ``omorf`` methods
"""
s = s.reshape(1, -1)
f = np.array(
[[self.objective['function'](self._remove_scaling(s.flatten()))]])
self.num_evals += 1
if self.f.size == 0:
self.S = s
self.f = f
else:
self.S = np.vstack((self.S, s))
self.f = np.vstack((self.f, f))
return np.asscalar(f)
def _generate_initial_set(self):
"""
Generates an initial set of samples using either coordinate directions or orthogonal, random directions
"""
if self.random_initial:
direcs = self._initial_random_directions(
self.p, self.bounds_l - self.s_old, self.bounds_u - self.s_old)
else:
direcs = self._initial_coordinate_directions(
self.p, self.bounds_l - self.s_old, self.bounds_u - self.s_old)
S = np.zeros((self.p, self.n))
f = np.zeros((self.p, 1))
S[0, :] = self.s_old
f[0, :] = self.f_old
for i in range(1, self.p):
del_s = np.minimum(
np.maximum(self.bounds_l - self.s_old, direcs[i, :]),
self.bounds_u - self.s_old)
S[i, :] = self.s_old + del_s
f[i, :] = self._blackbox_evaluation(self.s_old + del_s)
return S, f
def _initial_coordinate_directions(self, num_pnts, lower, upper):
"""
Generates coordinate directions
"""
at_lower_boundary = (lower > -0.01 * self.del_k)
at_upper_boundary = (upper < 0.01 * self.del_k)
direcs = np.zeros((num_pnts, self.n))
for i in range(1, num_pnts):
if 1 <= i < self.n + 1:
dirn = i - 1
step = self.del_k if not at_upper_boundary[
dirn] else -self.del_k
direcs[i, dirn] = step
elif self.n + 1 <= i < 2 * self.n + 1:
dirn = i - self.n - 1
step = -self.del_k
if at_lower_boundary[dirn]:
step = min(2.0 * self.del_k, upper[dirn])
if at_upper_boundary[dirn]:
step = max(-2.0 * self.del_k, lower[dirn])
direcs[i, dirn] = step
else:
itemp = (i - self.n - 1) // self.n
q = i - itemp * self.n - self.n
p = q + itemp
if p > self.n:
p, q = q, p - self.n
direcs[i, p - 1] = direcs[p, p - 1]
direcs[i, q - 1] = direcs[q, q - 1]
return direcs
def _initial_random_directions(self, num_pnts, lower, upper):
"""
Generates orthogonal, random directions
"""
direcs = np.zeros((self.n, max(2 * self.n + 1, num_pnts)))
idx_l = (lower == 0)
idx_u = (upper == 0)
active = np.logical_or(idx_l, idx_u)
inactive = np.logical_not(active)
nactive = np.sum(active)
ninactive = self.n - nactive
if ninactive > 0:
A = np.random.normal(size=(ninactive, ninactive))
Qred = np.linalg.qr(A)[0]
Q = np.zeros((self.n, ninactive))
Q[inactive, :] = Qred
for i in range(ninactive):
scale = self._get_scale(Q[:, i], self.del_k, lower, upper)
direcs[:, i] = scale * Q[:, i]
scale = self._get_scale(-Q[:, i], self.del_k, lower, upper)
direcs[:, self.n + i] = -scale * Q[:, i]
idx_active = np.where(active)[0]
for i in range(nactive):
idx = idx_active[i]
direcs[idx, ninactive + i] = 1.0 if idx_l[idx] else -1.0
direcs[:, ninactive +
i] = get_scale(direcs[:, ninactive + i], self.del_k, lower,
upper) * direcs[:, ninactive + i]
sign = 1.0 if idx_l[idx] else -1.0
if upper[idx] - lower[idx] > self.del_k:
direcs[idx, self.n + ninactive + i] = 2.0 * sign * self.del_k
else:
direcs[idx, self.n + ninactive +
i] = 0.5 * sign * (upper[idx] - lower[idx])
direcs[:, self.n + ninactive + i] = self._get_scale(
direcs[:, self.n + ninactive + i], 1.0, lower,
upper) * direcs[:, self.n + ninactive + i]
for i in range(num_pnts - 2 * self.n):
dirn = np.random.normal(size=(self.n, ))
for j in range(nactive):
idx = idx_active[j]
sign = 1.0 if idx_l[idx] else -1.0
if dirn[idx] * sign < 0.0:
dirn[idx] *= -1.0
dirn = dirn / np.linalg.norm(dirn)
scale = self._get_scale(dirn, self.del_k, lower, upper)
direcs[:, 2 * self.n + i] = dirn * scale
return np.vstack((np.zeros(self.n), direcs[:, :num_pnts].T))
@staticmethod
def _get_scale(dirn, delta, lower, upper):
scale = delta
for j in range(len(dirn)):
if dirn[j] < 0.0:
scale = min(scale, lower[j] / dirn[j])
elif dirn[j] > 0.0:
scale = min(scale, upper[j] / dirn[j])
return scale
@staticmethod
def _remove_point_from_set(S, f, s):
ind_current = np.where(
np.linalg.norm(S - s, axis=1, ord=np.inf) == 0.0)[0]
S = np.delete(S, ind_current, 0)
f = np.delete(f, ind_current, 0)
return S, f
def _choose_closest_points(self, n):
ind_closest = np.argsort(
np.linalg.norm(self.S - self.s_old, axis=1, ord=np.inf))[:n]
S = self.S[ind_closest, :]
f = self.f[ind_closest]
return S, f
def _remove_furthest_point(self, S, f):
ind_distant = np.argmax(
np.linalg.norm(S - self.s_old, axis=1, ord=np.inf))
S = np.delete(S, ind_distant, 0)
f = np.delete(f, ind_distant, 0)
return S, f
def _remove_points_outside_radius(self, S, f, radius):
ind_inside = np.where(
np.linalg.norm(S - self.s_old, axis=1, ord=np.inf) < radius)[0]
S = S[ind_inside, :]
f = f[ind_inside]
return S, f
def _update_bounds(self):
if self.bounds is not None:
self.bounds_l = np.maximum(np.zeros(self.n),
self.s_old - self.del_k)
self.bounds_u = np.minimum(np.ones(self.n),
self.s_old + self.del_k)
else:
self.bounds_l = self.s_old - self.del_k
self.bounds_u = self.s_old + self.del_k
self.s_c = 0.5 * (self.bounds_l + self.bounds_u)
return None
def _apply_scaling(self, S):
if self.bounds is not None:
shift = self.bounds[0].copy()
scale = self.bounds[1] - self.bounds[0]
return np.divide((S - shift), scale)
else:
return S
def _remove_scaling(self, S):
if self.bounds is not None:
shift = self.bounds[0].copy()
scale = self.bounds[1] - self.bounds[0]
return shift + np.multiply(S, scale)
else:
return S
def _sample_set(self, method, S=None, f=None, full_space=False):
if full_space:
q = self.p
else:
q = self.q
if method == 'replace':
S, f = self._remove_furthest_point(S, f)
elif method == 'improve':
S_hat = np.copy(S)
f_hat = np.copy(f)
S_hat, f_hat = self._remove_point_from_set(S_hat, f_hat,
self.s_old)
S_hat, f_hat = self._remove_furthest_point(S_hat, f_hat)
S = np.zeros((q, self.n))
f = np.zeros((q, 1))
S[0, :] = self.s_old
f[0, :] = self.f_old
S, f = self._LU_pivoting(S, f, S_hat, f_hat, full_space, 'improve')
elif method == 'new':
S_hat, f_hat = self._remove_points_outside_radius(
self.S, self.f, self.epsilon * self.del_k)
S_hat, f_hat = self._remove_point_from_set(S_hat, f_hat,
self.s_old)
S = np.zeros((q, self.n))
f = np.zeros((q, 1))
S[0, :] = self.s_old
f[0, :] = self.f_old
S, f = self._LU_pivoting(S, f, S_hat, f_hat, full_space)
return S, f
def _LU_pivoting(self, S, f, S_hat, f_hat, full_space, method=None):
phi_function, phi_function_deriv = self._get_phi_function_and_derivative(
S_hat, full_space)
if full_space:
psi = 1.0
q = self.p
else:
psi = 0.25
q = self.q
# Initialise U matrix of LU factorisation of M matrix (see Conn et al.)
U = np.zeros((q, q))
U[0, :] = phi_function(self.s_old)
# Perform the LU factorisation algorithm for the rest of the points
for k in range(1, q):
flag = True
v = np.zeros(q)
for j in range(k):
v[j] = -U[j, k] / U[j, j]
v[k] = 1.0
# If there are still points to choose from, find if points meet criterion. If so, use the index to choose
# point with given index to be next point in regression/interpolation set
if f_hat.size > 0:
M = np.absolute(
np.array([np.dot(phi_function(S_hat), v)]).flatten())
index = np.argmax(M)
if M[index] < 1.0e-4:
flag = False
elif method == 'improve':
if k == q - 1:
if M[index] < psi:
flag = False
else:
flag = False
# If index exists, choose the point with that index and delete it from possible choices
if flag:
s = S_hat[index, :]
S[k, :] = s
f[k, :] = f_hat[index]
S_hat = np.delete(S_hat, index, 0)
f_hat = np.delete(f_hat, index, 0)
# If index doesn't exist, solve an optimisation problem to find the point in the range which best satisfies criterion
else:
s = self._find_new_point(v, phi_function, phi_function_deriv,
full_space)
if f_hat.size > 0:
if M[index] >= abs(np.dot(v, phi_function(s))):
s = S_hat[index, :]
S[k, :] = s
f[k, :] = f_hat[index]
S_hat = np.delete(S_hat, index, 0)
f_hat = np.delete(f_hat, index, 0)
elif self.S.shape == np.unique(np.vstack((self.S, s)),
axis=0).shape:
rand = np.random.normal(size=self.n)
s = np.minimum(
np.maximum(
self.bounds_l, self.s_old +
self.del_k * rand / np.linalg.norm(rand)),
self.bounds_u)
if M[index] >= abs(np.dot(v, phi_function(s))):
s = S_hat[index, :]
S[k, :] = s
f[k, :] = f_hat[index]
S_hat = np.delete(S_hat, index, 0)
f_hat = np.delete(f_hat, index, 0)
else:
S[k, :] = s
f[k, :] = self._blackbox_evaluation(s)
else:
S[k, :] = s
f[k, :] = self._blackbox_evaluation(s)
else:
if self.S.shape == np.unique(np.vstack((self.S, s)),
axis=0).shape:
rand = np.random.normal(size=self.n)
s = np.minimum(
np.maximum(
self.bounds_l, self.s_old +
self.del_k * rand / np.linalg.norm(rand)),
self.bounds_u)
S[k, :] = s
f[k, :] = self._blackbox_evaluation(s)
else:
S[k, :] = s
f[k, :] = self._blackbox_evaluation(s)
# Update U factorisation in LU algorithm
phi = phi_function(s)
U[k, k] = np.dot(v, phi)
for i in range(k + 1, q):
U[k, i] += phi[i]
for j in range(k):
U[k, i] -= (phi[j] * U[j, i]) / U[j, j]
return S, f
def _get_phi_function_and_derivative(self, S_hat, full_space):
if self.method == 'trust-region':
Del_S = np.maximum(self.del_k,
np.linalg.norm(S_hat - self.s_c, axis=0))
def phi_function(s):
s_tilde = np.divide((s - self.s_c), Del_S)
try:
m, n = s_tilde.shape
except:
m = 1
s_tilde = s_tilde.reshape(1, -1)
phi = np.zeros((m, self.q))
for k in range(self.q):
phi[:, k] = np.prod(np.divide(
np.power(s_tilde, self.basis[k, :]),
factorial(self.basis[k, :])),
axis=1)
if m == 1:
return phi.flatten()
else:
return phi
def phi_function_deriv(s):
s_tilde = np.divide((s - self.s_c), Del_S)
phi_deriv = np.zeros((self.n, self.q))
for i in range(self.n):
for k in range(1, self.q):
if self.basis[k, i] != 0.0:
tmp = np.zeros(self.n)
tmp[i] = 1
phi_deriv[i, k] = self.basis[k, i] * np.prod(
np.divide(
np.power(s_tilde, self.basis[k, :] - tmp),
factorial(self.basis[k, :])))
return np.divide(phi_deriv.T, Del_S).T
elif self.method == 'omorf' and full_space:
Del_S = np.maximum(self.del_k,
np.linalg.norm(S_hat - self.s_c, axis=0))
def phi_function(s):
s_tilde = np.divide((s - self.s_c), Del_S)
try:
m, n = s_tilde.shape
except:
m = 1
s_tilde = s_tilde.reshape(1, -1)
phi = np.zeros((m, self.p))
phi[:, 0] = 1.0
phi[:, 1:] = s_tilde
if m == 1:
return phi.flatten()
else:
return phi
phi_function_deriv = None
elif self.method == 'omorf':
Del_S = np.linalg.norm(np.dot(S_hat - self.s_c, self.U), axis=0)
def phi_function(s):
u = np.divide(np.dot((s - self.s_c), self.U), Del_S)
try:
m, n = u.shape
except:
m = 1
u = u.reshape(1, -1)
phi = np.zeros((m, self.q))
for k in range(self.q):
phi[:,
k] = np.prod(np.divide(np.power(u, self.basis[k, :]),
factorial(self.basis[k, :])),
axis=1)
if m == 1:
return phi.flatten()
else:
return phi
def phi_function_deriv(s):
u = np.divide(np.dot((s - self.s_c), self.U), Del_S)
phi_deriv = np.zeros((self.d, self.q))
for i in range(self.d):
for k in range(1, self.q):
if self.basis[k, i] != 0.0:
tmp = np.zeros(self.d)
tmp[i] = 1
phi_deriv[i, k] = self.basis[k, i] * np.prod(
np.divide(np.power(u, self.basis[k, :] - tmp),
factorial(self.basis[k, :])))
phi_deriv = np.divide(phi_deriv.T, Del_S).T
return np.dot(self.U, phi_deriv)
return phi_function, phi_function_deriv
def _find_new_point(self,
v,
phi_function,
phi_function_deriv,
full_space=False):
bounds = []
for i in range(self.n):
bounds.append((self.bounds_l[i], self.bounds_u[i]))
if full_space:
c = v[1:]
res1 = optimize.linprog(c, bounds=bounds)
res2 = optimize.linprog(-c, bounds=bounds)
if abs(np.dot(v, phi_function(res1['x']))) > abs(
np.dot(v, phi_function(res2['x']))):
s = res1['x']
else:
s = res2['x']
else:
obj1 = lambda s: np.dot(v, phi_function(s))
jac1 = lambda s: np.dot(phi_function_deriv(s), v)
obj2 = lambda s: -np.dot(v, phi_function(s))
jac2 = lambda s: -np.dot(phi_function_deriv(s), v)
res1 = optimize.minimize(obj1, self.s_old, method='TNC', jac=jac1, \
bounds=bounds, options={'disp': False})
res2 = optimize.minimize(obj2, self.s_old, method='TNC', jac=jac2, \
bounds=bounds, options={'disp': False})
if abs(res1['fun']) > abs(res2['fun']):
s = res1['x']
else:
s = res2['x']
return s
def _build_model(self, S, f):
"""
Constructs quadratic model for ``trust-region`` or ``omorf`` methods
"""
if self.method == 'trust-region':
myParameters = [Parameter(distribution='uniform', lower=np.min(S[:,i]), \
upper=np.max(S[:,i]), order=2) for i in range(self.n)]
myBasis = Basis('total-order')
my_poly = Poly(myParameters, myBasis, method='least-squares', \
sampling_args={'sample-points':S, 'sample-outputs':f})
elif self.method == 'omorf':
Y = np.dot(S, self.U)
myParameters = [Parameter(distribution='uniform', lower=np.min(Y[:,i]), \
upper=np.max(Y[:,i]), order=2) for i in range(self.d)]
myBasis = Basis('total-order')
my_poly = Poly(myParameters, myBasis, method='least-squares', \
sampling_args={'sample-points':Y, 'sample-outputs':f})
my_poly.set_model()
return my_poly
def _compute_step(self, my_poly):
"""
Solves the trust-region subproblem for ``trust-region`` or ``omorf`` methods
"""
bounds = []
for i in range(self.n):
bounds.append((self.bounds_l[i], self.bounds_u[i]))
if self.method == 'trust-region':
res = optimize.minimize(lambda x: np.asscalar(my_poly.get_polyfit(x)), self.s_old, method='TNC', \
jac=lambda x: my_poly.get_polyfit_grad(x).flatten(), bounds=bounds, options={'disp': False})
elif self.method == 'omorf':
res = optimize.minimize(lambda x: np.asscalar(my_poly.get_polyfit(np.dot(x,self.U))), self.s_old, \
method='TNC', jac=lambda x: np.dot(self.U, my_poly.get_polyfit_grad(np.dot(x,self.U))).flatten(), \
bounds=bounds, options={'disp': False})
s_new = res.x
m_new = res.fun
return s_new, m_new
def _choose_best(self, S, f):
ind_min = np.argmin(f)
self.s_old = S[ind_min, :]
self.f_old = np.asscalar(f[ind_min])
return None
def _trust_region(self, s_old, del_k, del_min, eta1, eta2, gam1, gam2,
omega_s, max_evals, random_initial, epsilon):
"""
Computes optimum using the ``trust-region`` method
"""
self.n = s_old.size
self.s_old = self._apply_scaling(s_old)
if del_k is None:
if self.bounds is None:
self.del_k = 0.1 * max(np.linalg.norm(self.s_old, ord=np.inf),
1.0)
else:
self.del_k = 0.1
else:
self.del_k = del_k
self._update_bounds()
self.f_old = self._blackbox_evaluation(self.s_old)
self.q = int(comb(self.n + 2, 2))
self.p = int(comb(self.n + 2, 2))
self.random_initial = random_initial
self.epsilon = epsilon
Base = Basis('total-order', orders=np.tile([2], self.n))
self.basis = Base.get_basis()[:, range(self.n - 1, -1, -1)]
itermax = 10000
# Construct the sample set
S, f = self._generate_initial_set()
for i in range(itermax):
# print(self.s_old)
# print('-------------')
self._update_bounds()
if len(self.f) >= max_evals or self.del_k < del_min:
break
my_poly = self._build_model(S, f)
m_old = np.asscalar(my_poly.get_polyfit(self.s_old))
s_new, m_new = self._compute_step(my_poly)
# Safety step implemented in BOBYQA
if np.linalg.norm(s_new - self.s_old,
ord=np.inf) < omega_s * self.del_k:
S, f = self._sample_set('improve', S, f)
if max(np.linalg.norm(
S - self.s_old, axis=1,
ord=np.inf)) <= self.epsilon * self.del_k:
self.del_k *= gam1
continue
elif self.S.shape == np.unique(np.vstack((self.S, s_new)),
axis=0).shape:
ind_repeat = np.argmin(
np.linalg.norm(self.S - s_new, ord=np.inf, axis=1))
f_new = self.f[ind_repeat]
else:
f_new = self._blackbox_evaluation(s_new)
S = np.vstack((S, s_new))
f = np.vstack((f, f_new))
# Calculate trust-region factor
rho_k = (self.f_old - f_new) / (m_old - m_new)
self._choose_best(self.S, self.f)
self._update_bounds()
if len(self.f) >= max_evals or self.del_k < del_min:
break
if rho_k >= eta2:
S, f = self._sample_set('replace', S, f)
self.del_k *= gam2
elif rho_k >= eta1:
S, f = self._sample_set('replace', S, f)
else:
if max(np.linalg.norm(
S - self.s_old, axis=1,
ord=np.inf)) <= self.epsilon * self.del_k:
S, f = self._sample_set('improve', S, f)
self.del_k *= gam1
else:
S, f = self._sample_set('improve', S, f)
self.S = self._remove_scaling(self.S)
self._choose_best(self.S, self.f)
return self.s_old, self.f_old
def _omorf(self, s_old, del_k, del_min, eta1, eta2, gam1, gam2, omega_s,
max_evals, random_initial, epsilon, d, subspace_method):
"""
Computes optimum using the ``omorf`` method
"""
self.n = s_old.size
self.s_old = self._apply_scaling(s_old)
if del_k is None:
if self.bounds is None:
self.del_k = 0.1 * max(np.linalg.norm(self.s_old, ord=np.inf),
1.0)
else:
self.del_k = 0.1
else:
self.del_k = del_k
self._update_bounds()
self.f_old = self._blackbox_evaluation(self.s_old)
self.d = d
self.q = int(comb(self.d + 2, 2))
self.p = self.n + 1
self.random_initial = random_initial
self.subspace_method = subspace_method
self.epsilon = epsilon
Base = Basis('total-order', orders=np.tile([2], self.d))
self.basis = Base.get_basis()[:, range(self.d - 1, -1, -1)]
itermax = 10000
# Construct the sample set
S_full, f_full = self._generate_initial_set()
self._calculate_subspace(S_full, f_full)
S_red, f_red = self._sample_set('new')
for i in range(itermax):
# self._update_bounds()
if len(self.f) >= max_evals or self.del_k < del_min:
break
my_poly = self._build_model(S_red, f_red)
m_old = np.asscalar(my_poly.get_polyfit(np.dot(self.s_old,
self.U)))
s_new, m_new = self._compute_step(my_poly)
# Safety step implemented in BOBYQA
if np.linalg.norm(s_new - self.s_old,
ord=np.inf) < omega_s * self.del_k:
if max(np.linalg.norm(
S_full - self.s_old, axis=1,
ord=np.inf)) <= self.epsilon * self.del_k:
self._calculate_subspace(S_full, f_full)
S_red, f_red = self._sample_set('new')
self.del_k *= gam1
elif max(np.linalg.norm(
S_red - self.s_old, axis=1,
ord=np.inf)) <= self.epsilon * self.del_k:
S_full, f_full = self._sample_set('improve',
S_full,
f_full,
full_space=True)
self._calculate_subspace(S_full, f_full)
S_red, f_red = self._sample_set('new')
else:
S_red, f_red = self._sample_set('improve', S_red, f_red)
S_full, f_full = self._sample_set('improve',
S_full,
f_full,
full_space=True)
continue
if self.S.shape == np.unique(np.vstack((self.S, s_new)),
axis=0).shape:
ind_repeat = np.argmin(
np.linalg.norm(self.S - s_new, ord=np.inf, axis=1))
f_new = self.f[ind_repeat]
else:
f_new = self._blackbox_evaluation(s_new)
S_red = np.vstack((S_red, s_new))
f_red = np.vstack((f_red, f_new))
S_full = np.vstack((S_full, s_new))
f_full = np.vstack((f_full, f_new))
# Calculate trust-region factor
rho_k = (self.f_old - f_new) / (m_old - m_new)
self._choose_best(self.S, self.f)
self._update_bounds()
if len(self.f) >= max_evals or self.del_k < del_min:
break
if rho_k >= eta2:
S_red, f_red = self._sample_set('replace', S_red, f_red)
S_full, f_full = self._sample_set('replace', S_full, f_full)
self.del_k *= gam2
elif rho_k >= eta1:
S_red, f_red = self._sample_set('replace', S_red, f_red)
S_full, f_full = self._sample_set('replace', S_full, f_full)
else:
if max(np.linalg.norm(
S_full - self.s_old, axis=1,
ord=np.inf)) <= self.epsilon * self.del_k:
self._calculate_subspace(S_full, f_full)
S_red, f_red = self._sample_set('new')
self.del_k *= gam1
elif max(np.linalg.norm(
S_red - self.s_old, axis=1,
ord=np.inf)) <= self.epsilon * self.del_k:
S_full, f_full = self._sample_set('improve',
S_full,
f_full,
full_space=True)
self._calculate_subspace(S_full, f_full)
S_red, f_red = self._sample_set('new')
else:
S_red, f_red = self._sample_set('improve', S_red, f_red)
S_full, f_full = self._sample_set('improve',
S_full,
f_full,
full_space=True)
self.S = self._remove_scaling(self.S)
self._choose_best(self.S, self.f)
return self.s_old, self.f_old