def least_squares_test_III( f ):
eps = 2e-12
# define build points and values
num_dims = 2
num_pts_1d = 6
domain = TensorProductDomain( num_dims, [[-1.,1.]] )
x = -numpy.cos( numpy.linspace( 0., 1., num_pts_1d ) * numpy.pi );
[X,Y] = numpy.meshgrid( x, x )
build_points = numpy.vstack( ( X.reshape(( 1,X.shape[0]*X.shape[1])),
Y.reshape(( 1, Y.shape[0]*Y.shape[1]))))
#build_points = numpy.random.uniform( -1, 1, ( num_dims, 20 ) )
build_values = f( build_points )
# define solver
lamda = 0.
order = num_pts_1d - 1
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
pce = PCE( num_dims, order = order, basis = basis, func_domain = domain )
V = pce.vandermonde( build_points ).T
linear_solver = LeastSquaresSolver( lamda = lamda )
coeff, tmp = linear_solver.solve( V, build_values )
coeff = coeff[:,0]
residuals = build_values - numpy.dot( V, coeff )
# Initialise variables for the leave k out cross validation error
VtV_inv = numpy.linalg.inv( numpy.dot( V.T, V ) +
lamda * numpy.eye( V.shape[1] ) )
# perform cross validation on vandermonde matrix
num_folds = build_points.shape[1]
cv_iterator = KFoldCrossValidationIterator( num_folds,
build_points.shape[1],
seed = 1)
for train_indices, validation_indices in cv_iterator:
output = linear_solver.compute_residuals( V[train_indices],
build_values[train_indices],
V[validation_indices],
build_values[validation_indices] )
validation_residuals = output[2]
V_k = V[validation_indices]
H = numpy.eye( V_k.shape[0] ) - numpy.dot( V_k, numpy.dot( VtV_inv,
V_k.T ) )
assert numpy.linalg.cond( H ) <= 1. / numpy.finfo( numpy.double ).eps
H_inv = numpy.linalg.inv( H )
assert numpy.linalg.norm( numpy.dot( H_inv,
residuals[validation_indices] ) -
validation_residuals.reshape( V_k.shape[0] ) ) < eps
def test_pce_build( self ):
f = lambda x: numpy.sum( x**5, axis = 0 )
num_dims = 2
func_domain = TensorProductDomain( num_dims, ranges = [[-1.,1.]] )
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
predictor = PCE( num_dims, basis, order = 5, func_domain = func_domain,
index_norm_order = 0.5 )
rng = numpy.random.RandomState( 0 )
build_pts = rng.uniform( -1., 1., ( num_dims , 21 ) )
build_vals = f( build_pts )
predictor.build( build_pts, build_vals )
true_mean = mean_anisotropic_sum_of_monomials_2d( 5, 5 )
true_var = variance_anisotropic_sum_of_monomials_2d( 5, 5 )
assert numpy.allclose( predictor.mean(), true_mean )
assert numpy.allclose( predictor.variance(), true_var )
def interpolation_1d_test_I( f ):
seed = 2
rng = numpy.random#.RandomState( seed )
# 1D example with least interpolation
order = 3
num_dims = 1
func_domain = TensorProductDomain( num_dims, [[-1.,1.]] )
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
linear_solver = LeastInterpolation()
predictor = PCE( num_dims, basis, order = order,
linear_solver = linear_solver )
predictor.function_domain( func_domain )
x = -numpy.cos( numpy.linspace( 0., 1., order + 1 ) * numpy.pi );
build_points = x.reshape( 1, x.shape[0] )
build_values = f( build_points[0,:] )
p = predictor.build( build_points, build_values )
print 'c', predictor.coeff
def __init__( self, num_dims, max_level, quadrature_rule,
refinement_manager, sparse_grid_data, target_function ):
self.num_dims = num_dims
self.max_level = max_level
self.quadrature_rule = quadrature_rule
self.refinement_manager = refinement_manager
self.target_function = target_function
# for least interpolant
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
linear_solver = LeastInterpolation()
self.pce = PCE( self.num_dims, basis = basis,
linear_solver = linear_solver )
self.pce.function_domain( self.quadrature_rule.domain )
# grid points used in the sparse grid
self.grid_point_indices = set()
# number of grid points in the grid. This is not equal to
# len( self.grid_point_indices ). The later contains all grid points
# including candidates and those actually in the grid
self.num_grid_points = 0
# grid points that are candidates for addition to the sparse grid
# but have not been initialized and so can not be prioritized
self.uninit_grid_point_indices = set()
# grid points that are candidates for addition to the sparse grid
self.candidate_grid_point_indices_queue = []
# Python note: heap[0] has the smallest value ( inverse of priority ),
# i.e the highest prioirity
# grid points that are candidates for addition to the sparse grid
# This object will contain the same grid point indices as
# self.candidate_grid_point_indices_queue but is used to ensure a
# point is only added to the self.candidate_grid_point_indices_queue
#once
self.candidate_grid_point_indices_hash = set()
# keep track of the number of grid points generated
# ( self.num_grid_points + self.num_candidate_grid_points +
# self.num_uninit_grid_points )
self.num_grid_point_indices_generated = 0
self.data = sparse_grid_data
def test_omp_choloesky( self ):
f_1d = lambda x: x**10
num_dims = 1
order = 20
func_domain = TensorProductDomain( num_dims, [[-1,1]] )
build_pts = numpy.linspace(-.85,.9,14)
build_pts = numpy.atleast_2d( build_pts )
build_vals = f_1d( build_pts ).T
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
pce = PCE( num_dims, basis, order, func_domain )
all_train_indices = []
all_validation_indices = []
cv_iterator = LeaveOneOutCrossValidationIterator( build_pts.shape[1] )
for train_indices, validation_indices in cv_iterator:
all_train_indices.append( train_indices )
all_validation_indices.append( validation_indices )
vandermonde = pce.vandermonde( build_pts ).T
out = orthogonal_matching_pursuit_cholesky( vandermonde,
build_vals.squeeze(),
all_train_indices,
all_validation_indices,
0.0, 1000, 0 )
num_steps = out[1].shape[1]
# use num_steps -1 bscause leave one out cross validation is
# invalid when V is underdterimed which happens when i = num_steps.
for i in xrange( num_steps-1 ):
I = numpy.asarray( out[1][1,:i+1], dtype = numpy.int32 )
V = vandermonde[:,I]
for j in xrange( len( all_validation_indices ) ):
J = all_train_indices[j]
K = all_validation_indices[j]
A = V[J,:]
b = build_vals[J,:]
x = ridge_regression( A, b )
assert numpy.allclose( ( build_vals[K,0] -
numpy.dot( V, x )[K,0 ]),
out[2][i][j] )
all_train_indices = []
all_validation_indices = []
num_folds = 5
cv_iterator = KFoldCrossValidationIterator( num_folds,
build_pts.shape[1] )
for train_indices, validation_indices in cv_iterator:
all_train_indices.append( train_indices )
all_validation_indices.append( validation_indices )
vandermonde = pce.vandermonde( build_pts ).T
out = orthogonal_matching_pursuit_cholesky( vandermonde,
build_vals.squeeze(),
all_train_indices,
all_validation_indices,
0.0, 1000, 0 )
num_steps = out[1].shape[1]
for i in xrange( num_steps-1 ):
I = numpy.asarray( out[1][1,:i+1], dtype = numpy.int32 )
V = vandermonde[:,I]
for j in xrange( len( all_validation_indices ) ):
J = all_train_indices[j]
K = all_validation_indices[j]
A = V[J,:]
b = build_vals[J,:]
x = ridge_regression( A, b )
if ( len( I ) <= len( J ) ):
assert numpy.allclose( ( build_vals[K,0] -
numpy.dot( V, x )[K,0] ),
out[2][i][j] )
def xtest_grid_search_cross_validation( self ):
f_1d = lambda x: x**10
build_pts = numpy.linspace(-.85,.9,14)
build_pts = numpy.atleast_2d( build_pts )
build_vals = f_1d( build_pts ).T
# Test grid search cross validation when applied to Gaussian Process
num_dims = 1
func_domain = TensorProductDomain( num_dims, [[-1,1]] )
GP = GaussianProcess()
GP.set_verbosity( 0 )
GP.function_domain( func_domain )
loo_cv_iterator = LeaveOneOutCrossValidationIterator()
CV = GridSearchCrossValidation( loo_cv_iterator, GP )
CV.run( build_pts, build_vals )
I = numpy.arange( build_pts.shape[1] )
for i in xrange( build_pts.shape[1] ):
if i == 0 : J = I[1:]
elif i == build_pts.shape[1]-1 : J = I[:-1]
else: J = numpy.hstack( ( I[:i], I[i+1:] ) )
train_pts = build_pts[:,J]
train_vals = build_vals[J,:]
GP.build( train_pts, train_vals )
pred_vals = GP.evaluate_set( build_pts )
assert numpy.allclose( build_vals[i,0]-pred_vals[i],
CV.residuals[0][i] )
# Test grid search cross validation when applied to polynomial chaos
# expansions that are built using ridge regression
# The vandermonde matrix is built from scratch every time by the pce
num_dims = 1
order = 3
build_vals = f_1d( build_pts ).T
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
pce = PCE( num_dims, basis, order, func_domain )
loo_cv_iterator = LeaveOneOutCrossValidationIterator()
CV = GridSearchCrossValidation( loo_cv_iterator, pce )
CV.run( build_pts, build_vals )
I = numpy.arange( build_pts.shape[1] )
V = pce.vandermonde( build_pts ).T
for i in xrange( V.shape[0] ):
if i == 0 : J = I[1:]
elif i == build_pts.shape[1]-1 : J = I[:-1]
else: J = numpy.hstack( ( I[:i], I[i+1:] ) )
A = V[J,:]
b = build_vals[J,:]
x = ridge_regression( A, b )
assert numpy.allclose( (build_vals[i,0]-numpy.dot( V, x ))[i],
CV.residuals[0][i] )
# Test grid search cross validation when applied to polynomial chaos
# expansions that are built using ridge regression
# Specifying parse_cross_validation_data = True will ensure that
# the vandermonde matrix is not built from scratch every time by
# the pce
num_dims = 1
order = 3
build_vals = f_1d( build_pts ).T
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
pce = PCE( num_dims, basis, order, func_domain )
loo_cv_iterator = LeaveOneOutCrossValidationIterator()
CV = GridSearchCrossValidation( loo_cv_iterator, pce,
use_predictor_cross_validation = True)
CV.run( build_pts, build_vals )
I = numpy.arange( build_pts.shape[1] )
V = pce.vandermonde( build_pts ).T
for i in xrange( V.shape[0] ):
if i == 0 : J = I[1:]
elif i == build_pts.shape[1]-1 : J = I[:-1]
else: J = numpy.hstack( ( I[:i], I[i+1:] ) )
A = V[J,:]
b = build_vals[J,:]
x = ridge_regression( A, b )
assert numpy.allclose( (build_vals[i,0]-numpy.dot( V, x ))[i],
CV.residuals[0][i] )
# Test grid search cross validation when applied to polynomial chaos
# expansions that are built using ridge regression
# A closed form for the cross validation residual is used
num_dims = 1
order = 3
build_vals = f_1d( build_pts ).T
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
pce = PCE( num_dims, basis, order, func_domain )
loo_cv_iterator = LeaveOneOutCrossValidationIterator()
CV = GridSearchCrossValidation( loo_cv_iterator, pce,
use_predictor_cross_validation = True,
use_fast_predictor_cross_validation = True )
CV.run( build_pts, build_vals )
I = numpy.arange( build_pts.shape[1] )
V = pce.vandermonde( build_pts ).T
for i in xrange( V.shape[0] ):
if i == 0 : J = I[1:]
elif i == build_pts.shape[1]-1 : J = I[:-1]
else: J = numpy.hstack( ( I[:i], I[i+1:] ) )
A = V[J,:]
b = build_vals[J,:]
x = ridge_regression( A, b )
assert numpy.allclose( (build_vals[i,0]-numpy.dot( V, x ))[i],
CV.residuals[0][i] )
# Test grid search cross validation when applied to polynomial chaos
# expansions that are built using ridge regression
num_dims = 1
order = 3
build_vals = f_1d( build_pts ).T
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
pce = PCE( num_dims, basis, order, func_domain )
max_order = build_pts.shape[1]
orders = numpy.arange( 1, max_order )
lamda = numpy.array( [0.,1e-3,1e-2,1e-1] )
# note cartesian product takes type from first array in 1d sets
# so if I use orders first lamda will be rounded to 0
cv_params_grid_array = cartesian_product( [lamda,orders] )
cv_params_grid = []
for i in xrange( cv_params_grid_array.shape[0] ):
cv_params = {}
cv_params['lambda'] = cv_params_grid_array[i,0]
cv_params['order'] = numpy.int32( cv_params_grid_array[i,1] )
cv_params_grid.append( cv_params )
loo_cv_iterator = LeaveOneOutCrossValidationIterator()
CV = GridSearchCrossValidation( loo_cv_iterator, pce,
use_predictor_cross_validation = True,
use_fast_predictor_cross_validation = False )
CV.run( build_pts, build_vals, cv_params_grid )
k = 0
I = numpy.arange( build_pts.shape[1] )
for cv_params in cv_params_grid:
order = cv_params['order']
lamda = cv_params['lambda']
pce.set_order( order )
V = pce.vandermonde( build_pts ).T
for i in xrange( V.shape[0] ):
if i == 0 : J = I[1:]
elif i == build_pts.shape[1]-1 : J = I[:-1]
else: J = numpy.hstack( ( I[:i], I[i+1:] ) )
A = V[J,:]
b = build_vals[J,:]
x = ridge_regression( A, b, lamda = lamda )
assert numpy.allclose( ( build_vals[i,0]-
numpy.dot( V, x ) )[i],
CV.residuals[k][i] )
k += 1
print 'best',CV.best_cv_params
# Test grid search cross validation when applied to
# expansions that are built using a step based method
# ( LARS )
num_dims = 1
order = 3
build_vals = f_1d( build_pts ).T
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
pce = PCE( num_dims, basis, order, func_domain )
max_order = build_pts.shape[1]
orders = numpy.arange( 1, max_order )
lamda = numpy.array( [0.,1e-3,1e-2,1e-1] )
# note cartesian product takes type from first array in 1d sets
# so if I use orders first lamda will be rounded to 0
cv_params_grid_array = cartesian_product( [lamda,orders] )
cv_params_grid = []
for i in xrange( cv_params_grid_array.shape[0] ):
cv_params = {}
cv_params['solver'] = 4 # LARS
cv_params['order'] = numpy.int32( cv_params_grid_array[i,1] )
cv_params_grid.append( cv_params )
print cv_params_grid
loo_cv_iterator = LeaveOneOutCrossValidationIterator()
#loo_cv_iterator = KFoldCrossValidationIterator( 3 )
CV = GridSearchCrossValidation( loo_cv_iterator, pce,
use_predictor_cross_validation = True,
use_fast_predictor_cross_validation = False )
CV.run( build_pts, build_vals, cv_params_grid )
k = 0
I = numpy.arange( build_pts.shape[1] )
for cv_params in cv_params_grid:
order = cv_params['order']
pce.set_order( order )
V = pce.vandermonde( build_pts ).T
for i in xrange( V.shape[0] ):
if i == 0 : J = I[1:]
elif i == build_pts.shape[1]-1 : J = I[:-1]
else: J = numpy.hstack( ( I[:i], I[i+1:] ) )
A = V[J,:]
b = build_vals[J,:]
b = b.reshape( b.shape[0] )
x, metrics = least_angle_regression( A, b, 0., 4, 0., 1000,
0 )
assert numpy.allclose( ( build_vals[i,0]-
numpy.dot( V, x ) )[i],
CV.residuals[k][i] )
k += 1
#for i in xrange( len( CV.cv_params_set ) ):
# print CV.cv_params_set[i], CV.scores[i]
print 'best param', CV.best_cv_params
print 'best score', CV.best_score
print build_pts.shape[1]
# ( OMP )
num_dims = 1
order = 3
build_vals = f_1d( build_pts ).T
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
pce = PCE( num_dims, basis, order, func_domain )
max_order = build_pts.shape[1]
orders = numpy.arange( 1, max_order )
lamda = numpy.array( [0.,1e-3,1e-2,1e-1] )
# note cartesian product takes type from first array in 1d sets
# so if I use orders first lamda will be rounded to 0
cv_params_grid_array = cartesian_product( [lamda,orders] )
cv_params_grid = []
for i in xrange( cv_params_grid_array.shape[0] ):
cv_params = {}
cv_params['solver'] = 2 # OMP
cv_params['order'] = numpy.int32( cv_params_grid_array[i,1] )
cv_params_grid.append( cv_params )
print cv_params_grid
loo_cv_iterator = LeaveOneOutCrossValidationIterator()
#loo_cv_iterator = KFoldCrossValidationIterator( 3 )
CV = GridSearchCrossValidation( loo_cv_iterator, pce,
use_predictor_cross_validation = True,
use_fast_predictor_cross_validation = False )
CV.run( build_pts, build_vals, cv_params_grid )
k = 0
I = numpy.arange( build_pts.shape[1] )
for cv_params in cv_params_grid:
order = cv_params['order']
pce.set_order( order )
V = pce.vandermonde( build_pts ).T
for i in xrange( V.shape[0] ):
if i == 0 : J = I[1:]
elif i == build_pts.shape[1]-1 : J = I[:-1]
else: J = numpy.hstack( ( I[:i], I[i+1:] ) )
A = V[J,:]
b = build_vals[J,:]
b = b.reshape( b.shape[0] )
x, metrics = orthogonal_matching_pursuit( A, b, 0., 1000, 0 )
assert numpy.allclose( ( build_vals[i,0]-
numpy.dot( V, x ) )[i],
CV.residuals[k][i] )
k += 1
#for i in xrange( len( CV.cv_params_set ) ):
# print CV.cv_params_set[i], CV.scores[i]
print 'best param', CV.best_cv_params
print 'best score', CV.best_score
print build_pts.shape[1]
def test_polynomial_chaos_expansion( self ):
# test 1D bounded domain
num_dims = 1
func_domain = TensorProductDomain( num_dims )
poly_1d = [ JacobiPolynomial1D( 0., 0. ) ]
basis = TensorProductBasis( num_dims, poly_1d )
predictor = PCE( num_dims, basis, order = 2 )
predictor.set_coefficients( numpy.ones( predictor.coeff.shape ) )
predictor.function_domain( func_domain )
num_test_points = 20
test_points = \
numpy.linspace( 0., 1., num_test_points ).reshape(1,num_test_points)
pred_vals = predictor.evaluate_set( test_points )
x = 2. * test_points - 1.
test_vals = numpy.ones( num_test_points ) + x[0,:] + \
0.5 * ( 3.*x[0,:]**2 - 1. )
assert numpy.allclose( test_vals, pred_vals )
test_mean = 1.
test_variance = 1./3. + 1./5.
assert numpy.allclose( predictor.mean(), test_mean )
assert numpy.allclose( predictor.variance(), test_variance )
# test 2D bounded domain
num_dims = 2
func_domain = TensorProductDomain( num_dims )
poly_1d = [ JacobiPolynomial1D( 0., 0. ) ]
basis = TensorProductBasis( num_dims, poly_1d )
predictor = PCE( num_dims, basis, order = 2 )
predictor.set_coefficients( numpy.ones( predictor.coeff.shape ) )
predictor.function_domain( func_domain )
num_test_points = 20
test_points = numpy.random.uniform( 0., 1., ( num_dims, num_test_points))
pred_vals = predictor.evaluate_set( test_points )
x = 2. * test_points - 1.
test_vals = numpy.ones( num_test_points ) + x[0,:] + x[1,:] + \
0.5 * ( 3.*x[0,:]**2 - 1. ) + 0.5 * ( 3.*x[1,:]**2 - 1. ) + \
x[0,:] * x[1,:]
assert numpy.allclose( test_vals, pred_vals )
test_mean = 1.
test_variance = 2. * 1./3. + 1./9. + 2. * 1./5.
assert numpy.allclose( predictor.mean(), test_mean )
assert numpy.allclose( predictor.variance(), test_variance )
# test when domain is unbounded in one dimension
num_dims = 2
func_domain = TensorProductDomain( num_dims,
ranges = [[0.,1.],
[-numpy.inf, numpy.inf]] )
poly_1d = [ JacobiPolynomial1D( 0., 0. ), HermitePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
predictor = PCE( num_dims, basis, order = 2 )
predictor.set_coefficients( numpy.ones( predictor.coeff.shape ) )
predictor.function_domain( func_domain )
num_test_points = 20
x_1 = numpy.random.uniform( 0., 1., ( 1, 20 ) )
x_2 = numpy.random.normal( 0., 1., ( 1, 20 ) )
test_points = numpy.vstack( ( x_1, x_2 ) )
pred_vals = predictor.evaluate_set( test_points )
x = test_points
x[0,:] = 2. * x[0,:] - 1.
test_vals = numpy.ones( num_test_points ) + x[0,:] + x[1,:] + \
0.5 * ( 3.*x[0,:]**2 - 1. ) + ( x[1,:]**2 - 1. ) + \
x[0,:] * x[1,:]
assert numpy.allclose( test_vals, pred_vals )
test_mean = 1.
test_variance = 2. * 1./3. + 1./5. + 2. + 1.
assert numpy.allclose( predictor.mean(), test_mean )
assert numpy.allclose( predictor.variance(), test_variance )
class GridPointAdaptedSparseGrid(object):
def __init__( self, num_dims, max_level, quadrature_rule,
refinement_manager, sparse_grid_data, target_function ):
self.num_dims = num_dims
self.max_level = max_level
self.quadrature_rule = quadrature_rule
self.refinement_manager = refinement_manager
self.target_function = target_function
# for least interpolant
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
linear_solver = LeastInterpolation()
self.pce = PCE( self.num_dims, basis = basis,
linear_solver = linear_solver )
self.pce.function_domain( self.quadrature_rule.domain )
# grid points used in the sparse grid
self.grid_point_indices = set()
# number of grid points in the grid. This is not equal to
# len( self.grid_point_indices ). The later contains all grid points
# including candidates and those actually in the grid
self.num_grid_points = 0
# grid points that are candidates for addition to the sparse grid
# but have not been initialized and so can not be prioritized
self.uninit_grid_point_indices = set()
# grid points that are candidates for addition to the sparse grid
self.candidate_grid_point_indices_queue = []
# Python note: heap[0] has the smallest value ( inverse of priority ),
# i.e the highest prioirity
# grid points that are candidates for addition to the sparse grid
# This object will contain the same grid point indices as
# self.candidate_grid_point_indices_queue but is used to ensure a
# point is only added to the self.candidate_grid_point_indices_queue
#once
self.candidate_grid_point_indices_hash = set()
# keep track of the number of grid points generated
# ( self.num_grid_points + self.num_candidate_grid_points +
# self.num_uninit_grid_points )
self.num_grid_point_indices_generated = 0
self.data = sparse_grid_data
def build_root( self ):
"""
Initialize the grid by creating the root subspace
"""
root_grid_point_index = GridPointIndex( None )
self.insert_uninit_grid_point( root_grid_point_index )
def insert_remaining_candidate_points( self ):
if ( self.refinement_manager.candidate_point_needs_func_eval() ):
while ( len( self.candidate_grid_point_indices_queue ) > 0 ):
inv_priority, grid_point_index = \
heapq.heappop( self.candidate_grid_point_indices_queue )
self.candidate_grid_point_indices_hash.remove( grid_point_index )
self.insert_grid_point( grid_point_index )
def insert_next_grid_point( self ):
"""
Insert a subspace into the list of subspaces in the grid
"""
# if two indices have the same priority the indices themselves will be
# compared
inv_priority, grid_point_index = \
heapq.heappop( self.candidate_grid_point_indices_queue )
self.candidate_grid_point_indices_hash.remove( grid_point_index )
if ( grid_point_index not in self.grid_point_indices ):
if ( not self.refinement_manager.candidate_point_needs_func_eval() ):
self.evaluate_target_function( grid_point_index )
self.insert_grid_point( grid_point_index )
else:
# point was originally a missing parent
pass
return grid_point_index
def insert_uninit_grid_point( self, grid_point_index ):
if ( grid_point_index not in self.uninit_grid_point_indices and
grid_point_index not in self.candidate_grid_point_indices_hash and
grid_point_index not in self.grid_point_indices and
self.refinement_manager.grid_point_admissible( grid_point_index ) ):
self.uninit_grid_point_indices.add( grid_point_index )
grid_point_index.array_index = self.num_grid_point_indices_generated
self.num_grid_point_indices_generated += 1
def insert_candidate_grid_point( self, grid_point_index ):
# I think this can be removed if check is made when point is added to
# unitialized set, check for existance must be made for uninit
# candidate and sparse grid points
if ( grid_point_index not in self.candidate_grid_point_indices_hash ):
heapq.heappush( self.candidate_grid_point_indices_queue,
( 1. / grid_point_index.priority, grid_point_index ))
self.candidate_grid_point_indices_hash.add( grid_point_index )
else:
msg = 'grid point already exists'
raise Exception, msg
def insert_grid_point( self, grid_point_index ):
self.grid_point_indices.add( grid_point_index )
self.num_grid_points += 1
def refine( self ):
"""
Perform point-wise adaptive refinement.
Extract the grid points with the highest priority.
"""
grid_point_index = self.insert_next_grid_point()
if self.refinement_manager.grid_point_warrants_refinement( grid_point_index ):
for dim in xrange( self.quadrature_rule.num_dims ):
# todo would be better not to assume that a left and right child
# are both available or more are not as well
child_index = self.quadrature_rule.child_index( grid_point_index,
dim, 'left' )
if child_index is not None:
self.insert_uninit_grid_point( child_index )
child_index = self.quadrature_rule.child_index( grid_point_index,
dim, 'right' )
if child_index is not None:
self.insert_uninit_grid_point( child_index )
def compute_hierarical_surpluses( self, fn_val, coord, grid_point_index ):
self.insert_missing_parent_grid_points( grid_point_index )
subspace_index = grid_point_index.subspace_index()
return fn_val - self.evaluate_set( coord.reshape( self.num_dims,1),
subspace_index )
# evaluate all points belonging to subspaces that have indices <=
# current_subspace_index
def evaluate_set( self, pts, current_subspace_index = None ):
result = numpy.zeros( ( pts.shape[1] ), numpy.double )
for grid_point_index in self.grid_point_indices:
evaluate_grid_point = True
# this is very expensive. at least in python
#grid_point_subspace_index = grid_point_index.subspace_index()
#if ( current_subspace_index is None ):
# evaluate_grid_point = True
#elif ( grid_point_subspace_index <= current_subspace_index ):
# evaluate_grid_point = True
#else:
# evaluate_grid_point = False
if ( evaluate_grid_point ):
if ( grid_point_index.data is not None ):
#evaluate basis
result += \
self.data.hier_surplus[grid_point_index.array_index] * \
self.quadrature_rule.basis.value( pts, grid_point_index,
self.quadrature_rule.domain )
else:
result += \
self.data.hier_surplus[grid_point_index.array_index] * \
numpy.ones( ( result.shape[0] ), numpy.double )
return result
def evaluate_set1( self, pts, current_subspace_index = None ):
if ( len( self.grid_point_indices ) == 0 ):
return numpy.zeros( ( pts.shape[1] ), numpy.double )
poly_indices = \
get_polynomial_indices_of_subspace_indices(
self.refinement_manager.subspace_indices,
self.quadrature_rule )
coords = numpy.empty( ( self.num_dims, len( self.grid_point_indices ) ),
numpy.double )
values = numpy.empty( ( len( self.grid_point_indices ), 1 ),
numpy.double )
for i, grid_point_index in enumerate( self.grid_point_indices ):
coords[:,i] = \
self.quadrature_rule.coordinate( grid_point_index )
values[i,0] = self.data.fn_vals[grid_point_index.array_index]
self.pce.build( coords, values )
return self.pce.evaluate_set( pts )
def grid_point_coordinates( self ):
coords = numpy.empty( ( self.num_dims, len( self.grid_point_indices ) ),
numpy.double )
for i, grid_point_index in enumerate( self.grid_point_indices ):
coords[:,i] = \
self.quadrature_rule.coordinate( grid_point_index )
return coords
def insert_parent_grid_point( self, grid_point_index ):
# If the parent is missing from self.sparse_grid_indices
# always promote it to self.sparse grid_indices.
# If the parent is in self.candidate_grid_point_indices
# keep it there so it will be refined. Doing this means that
# in insert_next_grid_point we must include a check that does not
# allow points that are already in self.sparse_grid_indices to
# be added again. We must also evaluate
# the target function if it has not been evaluated already.
# If the parent is in self.uninit_grid_point_indices
# then promote it to the candidate set. We must always calculate
# the priority and evaluate the function if not done so already.
self.insert_missing_parent_grid_points( grid_point_index )
# Check if grid_point_index is in a candidate point
array_index = None
grid_point_index_ref=find( self.candidate_grid_point_indices_hash,
grid_point_index )
if ( grid_point_index_ref is not None ):
# grid_point_index is not a candidate point.
# grid_point_index_ref contains is the original index
# contained in self.candidate_grid_point_indices_hash.
# grid_point_index will not have priority or aray_index set.
# only evaluate function if not done so already
if ( not self.refinement_manager.candidate_point_needs_func_eval() ):
self.evaluate_target_function( grid_point_index_ref )
# promote point to the sparse grid
self.insert_grid_point( grid_point_index_ref )
else:
# grid_point_index is not a candidate point so check if it is
# an uninit point
grid_point_index_ref = \
find( self.uninit_grid_point_indices, grid_point_index )
if ( grid_point_index_ref is not None ):
# grid_point_index is an uninit point
# grid_point_index_ref contains is the original index
# contained in self.uninit_grid_point_indices.
self.uninit_grid_point_indices.remove(grid_point_index_ref)
self.insert_candidate_grid_point( grid_point_index_ref )
else:
# grid_point_index is not an uninit point or candidate
# point
grid_point_index_ref = grid_point_index
grid_point_index_ref.array_index = \
self.num_grid_point_indices_generated
self.num_grid_point_indices_generated += 1
self.evaluate_target_function( grid_point_index_ref )
self.refinement_manager.prioritize_grid_point( grid_point_index_ref )
self.insert_candidate_grid_point( grid_point_index_ref )
self.insert_grid_point( grid_point_index )
def insert_missing_parent_grid_points( self, grid_point_index ):
for d in xrange( grid_point_index.num_effect_dims() ):
dim = grid_point_index.dimension_from_array_index ( d )
parent_index = self.quadrature_rule.parent_index( grid_point_index,
dim )
if ( parent_index not in self.grid_point_indices ):
self.insert_parent_grid_point( parent_index )
def evaluate_target_function( self, grid_point_index ):
coord = self.quadrature_rule.coordinate( grid_point_index )
coord = coord.reshape( coord.shape[0], 1 )
array_index = grid_point_index.array_index
self.data.set_fn_vals( self.target_function( coord ), array_index )
hier_surplus = \
self.compute_hierarical_surpluses( self.data.fn_vals[array_index],
coord, grid_point_index )
self.data.set_hier_surplus( hier_surplus, array_index )
self.refinement_manager.num_model_runs +=1
def prioritize( self ):
for grid_point_index in self.uninit_grid_point_indices:
if ( self.refinement_manager.candidate_point_needs_func_eval() ):
self.evaluate_target_function( grid_point_index )
self.refinement_manager.prioritize_grid_point( grid_point_index )
self.insert_candidate_grid_point( grid_point_index )
self.uninit_grid_point_indices = set()
def pce_study( build_pts, build_vals, domain,
test_pts, test_vals,
results_file = None,
cv_file = None, solver_type = 2 ):
num_dims = build_pts.shape[0]
index_generator = IndexGenerator()
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
pce = PCE( num_dims, order = 0, basis = basis, func_domain = domain )
if ( solver_type == 1 ):
num_folds = build_pts.shape[1]
else:
num_folds = 20
index_norm_orders = numpy.linspace( 0.4, 1.0, 4 )
#if (solver_tupe == 1):
# index_norm_orders = [.4,.5,.6,.7,.8,.9,1.]
#solvers = numpy.array( [solver_type], numpy.int32 )
#cv_params_grid_array = cartesian_product( [solvers,orders] )
cv_params_grid = []
for index_norm_order in index_norm_orders:
level = 2
# determine what range of orders to consider.
# spefically consider any order that results in a pce with terms <= 3003
while ( True ):
#index_generator.set_parameters( num_dims, level,
# index_norm_order = index_norm_order)
indices = index_generator.get_isotropic_indices( num_dims, level,
index_norm_order )
num_indices = len( indices )
print level, index_norm_order, len ( indices )
if ( num_indices > 3003 ):
break
cv_params = {}
cv_params['solver'] = solver_type
cv_params['order'] = level
cv_params['index_norm_order'] = index_norm_order
if ( cv_params['solver'] > 1 or
num_indices <= build_pts.shape[1] ):
# only do least squares on over-determined systems
cv_params_grid.append( cv_params )
level += 1
print cv_params_grid
# cv_iterator = LeaveOneOutCrossValidationIterator()
cv_iterator = KFoldCrossValidationIterator( num_folds = num_folds )
CV = GridSearchCrossValidation( cv_iterator, pce,
use_predictor_cross_validation = True,
use_fast_predictor_cross_validation = True )
t0 = time.time()
CV.run( build_pts, build_vals, cv_params_grid )
time_taken = time.time() - t0
print 'cross validation took ', time_taken, ' seconds'
print "################"
print "Best cv params: ", CV.best_cv_params
print "Best cv score: ", CV.best_score
print "################"
#for i in xrange( len( CV.cv_params_set ) ):
# print CV.cv_params_set[i], CV.scores[i]
best_order = CV.best_cv_params['order']
best_index_norm_order = CV.best_cv_params['index_norm_order']
best_pce = PCE( num_dims,
order = best_order,
basis = basis,
func_domain = domain,
index_norm_order = best_index_norm_order)
V = best_pce.vandermonde( build_pts ).T
best_pce.set_solver( CV.best_cv_params['solver'] )
if cv_params['solver'] != 1 and cv_params['solver'] != 5:
best_res_tol = CV.best_cv_params['norm_residual']
best_pce.linear_solver.residual_tolerance = best_res_tol
sols, sol_metrics = best_pce.linear_solver.solve( V, build_vals )
coeff = sols[:,-1]
best_pce.set_coefficients( coeff )
error = abs( build_vals - best_pce.evaluate_set( build_pts ) )
print max( error )
print 'Evaluating best pce at test points'
num_test_pts = test_pts.shape[1]
pce_vals_pred = best_pce.evaluate_set( test_pts ).T
print test_vals.shape, pce_vals_pred.shape
error = test_vals.squeeze() - pce_vals_pred
linf_error = numpy.max( numpy.absolute( error ) )
l2_error = numpy.sqrt( numpy.dot( error.T, error ) / num_test_pts )
mean = numpy.mean( pce_vals_pred )
var = numpy.var( pce_vals_pred )
pce_mean = best_pce.mean()
pce_var = best_pce.variance()
if results_file is not None:
results_file.write( '%1.15e' %linf_error + ',' + '%1.15e' %l2_error +
',' + '%1.15e' %mean + ',' + '%1.15e' %var +
',%1.15e' %pce_mean + ',' + '%1.15e' %pce_var + '\n')
print "linf error: ", linf_error
print "l2 error: ", l2_error
print "mean: ", mean
print "var: ", var
print "pce mean: ", pce_mean
print "pce var: ", pce_var
def cv_vs_error_study( build_pts, build_vals, domain,
test_pts, test_vals,
results_file = None,
cv_file = None, solver_type = 2 ):
num_dims = build_pts.shape[0]
if ( num_dims == 10 ):
max_order = 5
elif ( num_dims == 15 ):
max_order = 4
else:
max_order = 3
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
pce = PCE( num_dims, order = 0, basis = basis, func_domain = domain )
orders = numpy.arange( 1, max_order + 1 )
solvers = numpy.array( [solver_type], numpy.int32 )
cv_params_grid_array = cartesian_product( [solvers,orders] )
cv_params_grid = []
for i in xrange( cv_params_grid_array.shape[0] ):
cv_params = {}
cv_params['solver'] = numpy.int32( cv_params_grid_array[i,0] )
cv_params['order'] = numpy.int32( cv_params_grid_array[i,1] )
num_pce_terms = polynomial_space_dimension( num_dims,
cv_params['order'] )
if ( cv_params['solver'] <= 1 and
num_pce_terms >= build_pts.shape[1] ):
cv_params['lambda'] = 1.e-12
cv_params_grid.append( cv_params )
# print cv_params_grid
# cv_iterator = LeaveOneOutCrossValidationIterator()
cv_iterator = KFoldCrossValidationIterator( num_folds = 20 )
CV = GridSearchCrossValidation( cv_iterator, pce,
use_predictor_cross_validation = True,
use_fast_predictor_cross_validation = True )
t0 = time.time()
CV.run( build_pts, build_vals, cv_params_grid )
time_taken = time.time() - t0
print 'cross validation took ', time_taken, ' seconds'
print "################"
print "Best cv params: ", CV.best_cv_params
print "Best cv score: ", CV.best_score
print "################"
for order in orders:
residual_norms = numpy.empty( len( CV.cv_params_set ), numpy.double )
scores = numpy.empty( len( CV.cv_params_set ), numpy.double )
k = 0
for i in xrange( len( CV.cv_params_set ) ):
if ( CV.cv_params_set[i]['order'] == order ):
residual_norms[k] = CV.cv_params_set[i]['norm_residual']
scores[k] = CV.scores[i]
k += 1
residual_norms.resize( k )
scores.resize( k )
pce = PCE( num_dims,
order = order,
basis = basis,
func_domain = domain )
V = pce.vandermonde( build_pts ).T
pce.set_solver( CV.best_cv_params['solver'] )
# pce.linear_solver.max_iterations = 3
sols, sol_metrics = pce.linear_solver.solve( V, build_vals )
from sklearn.linear_model import orthogonal_mp
l2_error = numpy.empty( ( sols.shape[1] ), numpy.double )
residuals = numpy.empty( ( sols.shape[1] ), numpy.double )
test_pts = numpy.random.uniform( 0., 1., ( num_dims, 1000 ) )
f = GenzModel( domain, 'oscillatory' )
# f.set_coefficients( 4.5, 'no-decay' )
f.set_coefficients( 4.5, 'quadratic-decay' )
test_vals = f( test_pts ).reshape( ( test_pts.shape[1], 1 ) )
for i in xrange( sols.shape[1] ):
coeff = sols[:,i]
pce.set_coefficients( coeff )
residuals[i] = numpy.linalg.norm( build_vals -
pce.evaluate_set( build_pts ) )
num_test_pts = test_pts.shape[1]
pce_vals_pred = pce.evaluate_set( test_pts ).T
error = test_vals.squeeze() - pce_vals_pred
l2_error[i] = numpy.linalg.norm( error ) / numpy.sqrt( num_test_pts )
import pylab
print residuals, l2_error
print residual_norms, scores
pylab.loglog( residuals, l2_error, label = str( order ) + 'true' )
pylab.loglog( residual_norms, scores, label = str( order )+'-cv' )
pylab.xlim([1e-3,10])
pylab.legend()
pylab.show()
def pce_study( build_pts, build_vals, domain,
test_pts, test_vals,
results_file = None,
cv_file = None, solver_type = 2 ):
num_dims = build_pts.shape[0]
index_generator = IndexGenerator()
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
pce = PCE( num_dims, order = 0, basis = basis, func_domain = domain )
if ( solver_type == 1 ):
num_folds = build_pts.shape[1]
else:
num_folds = 20
index_norm_orders = numpy.linspace( 0.4, 1.0, 4 )
#solvers = numpy.array( [solver_type], numpy.int32 )
#cv_params_grid_array = cartesian_product( [solvers,orders] )
cv_params_grid = []
for index_norm_order in index_norm_orders:
level = 2
# determine what range of orders to consider.
# spefically consider any order that results in a pce with terms <= 3003
while ( True ):
index_generator.set_parameters( num_dims, level,
index_norm_order = index_norm_order )
index_generator.build_isotropic_index_set()
print level, index_norm_order, index_generator.num_indices
if ( index_generator.num_indices > 3003 ):
break
cv_params = {}
cv_params['solver'] = solver_type
cv_params['order'] = level
cv_params['index_norm_order'] = index_norm_order
if ( cv_params['solver'] > 1 or
index_generator.num_indices <= build_pts.shape[1] ):
# only do least squares on over-determined systems
cv_params_grid.append( cv_params )
else:
break
level += 1
print cv_params_grid
# cv_iterator = LeaveOneOutCrossValidationIterator()
cv_iterator = KFoldCrossValidationIterator( num_folds = num_folds )
CV = GridSearchCrossValidation( cv_iterator, pce,
use_predictor_cross_validation = True,
use_fast_predictor_cross_validation = True )
t0 = time.time()
CV.run( build_pts, build_vals, cv_params_grid )
time_taken = time.time() - t0
print 'cross validation took ', time_taken, ' seconds'
print "################"
print "Best cv params: ", CV.best_cv_params
print "Best cv score: ", CV.best_score
print "################"
#for i in xrange( len( CV.cv_params_set ) ):
# print CV.cv_params_set[i], CV.scores[i]
best_order = CV.best_cv_params['order']
best_index_norm_order = CV.best_cv_params['index_norm_order']
best_pce = PCE( num_dims,
order = best_order,
basis = basis,
func_domain = domain,
index_norm_order = best_index_norm_order)
V = best_pce.vandermonde( build_pts ).T
best_pce.set_solver( CV.best_cv_params['solver'] )
if cv_params['solver'] > 1 :
best_res_tol = CV.best_cv_params['norm_residual']
best_pce.linear_solver.residual_tolerance = best_res_tol
sols, sol_metrics = best_pce.linear_solver.solve( V, build_vals )
coeff = sols[:,-1]
best_pce.set_coefficients( coeff )
error = abs( build_vals - best_pce.evaluate_set( build_pts ) )
print max( error )
print 'Evaluating best pce at test points'
num_test_pts = test_pts.shape[1]
pce_vals_pred = best_pce.evaluate_set( test_pts ).T
print test_vals.shape, pce_vals_pred.shape
error = test_vals.squeeze() - pce_vals_pred
linf_error = numpy.max( numpy.absolute( error ) )
l2_error = numpy.sqrt( numpy.dot( error.T, error ) / num_test_pts )
mean = numpy.mean( pce_vals_pred )
var = numpy.var( pce_vals_pred )
pce_mean = best_pce.mean()
pce_var = best_pce.variance()
if results_file is not None:
results_file.write( '%1.15e' %linf_error + ',' + '%1.15e' %l2_error +
',' + '%1.15e' %mean + ',' + '%1.15e' %var +
',%1.15e' %pce_mean + ',' + '%1.15e' %pce_var + '\n')
print "linf error: ", linf_error
print "l2 error: ", l2_error
print "mean: ", mean
print "var: ", var
print "pce mean: ", pce_mean
print "pce var: ", pce_var
me, te, ie = best_pce.get_sensitivities()
interaction_values, interaction_terms = best_pce.get_interactions()
show = False
fignum = 1
filename = 'oscillator-individual-interactions.png'
plot_interaction_values( interaction_values, interaction_terms, title = 'Sobol indices', truncation_pct = 0.95, filename = filename, show = show,
fignum = fignum )
fignum += 1
filename = 'oscillator-dimension-interactions.png'
plot_interaction_effects( ie, title = 'Dimension-wise joint effects', truncation_pct = 0.95, filename = filename, show = show,fignum = fignum )
fignum += 1
filename = 'oscillator-main-effects.png'
plot_main_effects( me, truncation_pct = 0.95, title = 'Main effect sensitivity indices', filename = filename, show = show, fignum = fignum )
fignum += 1
filename = 'oscillator-total-effects.png'
plot_total_effects( te, truncation_pct = 0.95, title = 'Total effect sensitivity indices', filename = filename, show = show, fignum = fignum )
fignum += 1
from scipy.stats.kde import gaussian_kde
pylab.figure( fignum )
pce_kde = gaussian_kde( pce_vals_pred )
pce_kde_x = numpy.linspace( pce_vals_pred.min(), pce_vals_pred.max(), 100 )
pce_kde_y = pce_kde( pce_kde_x )
pylab.plot( pce_kde_x, pce_kde_y,label = 'pdf of surrogate' )
true_kde = gaussian_kde( test_vals )
true_kde_x = numpy.linspace( test_vals.min(), test_vals.max(), 100 )
true_kde_y = true_kde( true_kde_x )
pylab.plot( true_kde_x, true_kde_y, label = 'true pdf' )
pylab.legend(loc=2)
pylab.show()
def test_least_interpolant( self ):
num_dims = 2
func_domain = TensorProductDomain( num_dims, [[-1,1]] )
x = -numpy.cos( numpy.linspace( 0., 1., 10 ) * numpy.pi );
[X,Y] = numpy.meshgrid( x, x )
build_points = numpy.vstack( ( X.reshape(( 1,X.shape[0]*X.shape[1])),
Y.reshape(( 1, Y.shape[0]*Y.shape[1]))))
build_values = matlab_peaks( build_points )
poly_1d = [ LegendrePolynomial1D() ]
basis = TensorProductBasis( num_dims, poly_1d )
linear_solver = LeastInterpolation()
predictor = PCE( build_points.shape[0], basis = basis,
linear_solver = linear_solver )
predictor.function_domain( func_domain )
predictor.build( build_points, build_values )
result = predictor.evaluate_set( build_points )
if ( result.ndim > 1 ): result = result.reshape( result.shape[0] )
error = result - build_values
l2_error = numpy.sqrt( numpy.dot( error.T, error ) / \
build_points.shape[1] )
print 'error at nodes: %1.15e' %l2_error
assert l2_error < self.eps
poly_1d = [ JacobiPolynomial1D( 0.5, 0.5 ) ]
basis = TensorProductBasis( num_dims, poly_1d )
linear_solver = LeastInterpolation()
predictor = PCE( build_points.shape[0], basis = basis,
linear_solver = linear_solver )
predictor.function_domain( func_domain )
predictor.build( build_points, build_values )
result = predictor.evaluate_set( build_points )
if ( result.ndim > 1 ): result = result.reshape( result.shape[0] )