def test_gp_addative_kernel(self):
"""Test Gaussian process predictions with the addative kernel."""
train_features, train_targets, test_features, test_targets = get_data()
# Test prediction with addative linear and gaussian kernel.
kdict = {
'k1': {
'type': 'linear',
'features': [0, 1],
'scaling': 1.
},
'k2': {
'type': 'gaussian',
'features': [2, 3],
'width': 1.,
'scaling': 1.
},
'c1': {
'type': 'constant',
'const': 1.
}
}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kdict,
regularization=1e-3,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
self.assertEqual(len(pred['prediction']), len(test_features))
print('addition prediction:', pred['validation_error']['rmse_average'])
def test_cinv_is_good(self):
kdict = [{'type': 'gaussian', 'width': width, 'scaling': scaling}]
gp = GaussianProcess(
kernel_list=kdict, train_fp=train, train_target=target,
gradients=None, scale_data=True,
optimize_hyperparameters=False)
gp_grad = GaussianProcess(
kernel_list=kdict, train_fp=train, train_target=target,
gradients=gradients, scale_data=True,
optimize_hyperparameters=False)
print('Checking size of covariance matrix (K).')
self.assertEqual(np.shape(gp.cinv), (len(train), len(train)))
print('Checking size of covarience matrix with gradients (Ktilde).')
self.assertEqual(np.shape(gp_grad.cinv),
(len(train) + len(train) * np.shape(train)[1],
len(train) + len(train) * np.shape(train)[1]))
print('Comparing bigK and bigKtilde.')
cov = np.linalg.inv(gp.cinv)
cov_grad = np.linalg.inv(gp_grad.cinv)
np.testing.assert_array_almost_equal(
cov, cov_grad[0:len(train), 0:len(train)], decimal=10)
def test_gp_laplacian_kernel(self):
"""Test Gaussian process predictions with the laplacian kernel."""
train_features, train_targets, test_features, test_targets = get_data()
# Test prediction routine with laplacian kernel.
kdict = [{
'type': 'laplacian',
'width': 1.,
'scaling': 1.,
'bounds': ((1e-5, None), ) * np.shape(train_features)[1],
'scaling_bounds': ((0., None), )
}]
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kdict,
regularization=np.sqrt(1e-3),
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
self.assertEqual(len(pred['prediction']), len(test_features))
print('laplacian prediction:',
pred['validation_error']['rmse_average'])
def test_gp_multiplication_kernel(self):
"""Test Gaussian process predictions with the multiplication kernel."""
train_features, train_targets, test_features, test_targets = get_data()
# Test prediction with multiplication of linear & gaussian kernel.
kdict = [{
'type': 'linear',
'features': [0, 1],
'scaling': 1.
}, {
'type': 'gaussian',
'features': [2, 3],
'width': 1.,
'scaling': 1.,
'operation': 'multiplication'
}, {
'type': 'constant',
'const': 1.
}]
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kdict,
regularization=np.sqrt(1e-3),
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
self.assertEqual(len(pred['prediction']), len(test_features))
print('multiplication prediction:',
pred['validation_error']['rmse_average'])
def test_gp(self):
"""Test Gaussian process predictions with rescaling."""
train_features, train_targets, test_features, test_targets = get_data()
# Test prediction routine with gaussian kernel.
kdict = {'k1': {'type': 'gaussian', 'width': 1., 'scaling': 1.}}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kdict,
regularization=1e-3,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True,
uncertainty=True)
opt = gp.kernel_dict['k1']['width']
orig = hs.rescale_hyperparameters(gp.scaling,
gp.kernel_dict)['k1']['width']
assert not np.allclose(opt, orig)
scaled = hs.hyperparameters(gp.scaling, gp.kernel_dict)['k1']['width']
assert np.allclose(opt, scaled)
print('gaussian prediction (rmse):',
pred['validation_error']['rmse_average'])
def test_gp_quadratic_kernel(self):
"""Test Gaussian process predictions with the quadratic kernel."""
train_features, train_targets, test_features, test_targets = get_data()
# Test prediction routine with quadratic kernel.
kdict = {
'k1': {
'type': 'quadratic',
'slope': 1.,
'degree': 1.,
'scaling': 1.,
'bounds': ((1e-5, None), ) * (np.shape(train_features)[1] + 1),
'scaling_bounds': ((0., None), )
}
}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kdict,
regularization=1e-3,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
self.assertEqual(len(pred['prediction']), len(test_features))
print('quadratic prediction:',
pred['validation_error']['rmse_average'])
def prediction(self, train_features, train_targets, test_features,
test_targets):
"""Ridge regression predictions."""
# Test ridge regression predictions.
sigma = 1.
kdict = [{'type': 'gaussian', 'width': sigma, 'dimension': 'single'}]
regularization = np.sqrt(0.001)
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kdict,
regularization=regularization,
optimize_hyperparameters=False,
scale_data=True)
output = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True,
uncertainty=True)
r = [
np.shape(train_features)[0],
output['validation_error']['rmse_average'],
output['validation_error']['absolute_average'],
np.mean(output['uncertainty'])
]
return r
def train_gaussian_process(self, train_features, train_targets):
"""Generate a general gaussian process model.
Parameters
----------
train_features : array
The array of training features.
train_targets : array
A list of training target values.
Returns
-------
gp : object
The trained Gaussian process.
"""
train_features, train_targets = self._process_train_data(
train_features, train_targets)
if 'smooth' in self.kernel:
kdict = smooth_kernel(train_features, self.dimension)
else:
kdict = general_kernel(train_features, self.dimension)
self.gp = GaussianProcess(
train_fp=train_features, train_target=train_targets,
kernel_list=kdict, regularization=1e-2,
optimize_hyperparameters=True, scale_data=False
)
return self.gp
def test_gp_linear_kernel(self):
"""Test Gaussian process predictions with the linear kernel."""
train_features, train_targets, test_features, test_targets = get_data()
# Test prediction routine with linear kernel.
kdict = {
'k1': {
'type': 'linear',
'scaling': 1.,
'scaling_bounds': ((0., None), )
},
'c1': {
'type': 'constant',
'const': 1.
}
}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kdict,
regularization=1e-3,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
self.assertEqual(len(pred['prediction']), len(test_features))
print('linear prediction:', pred['validation_error']['rmse_average'])
def test_acquisition(self):
"""Test acquisition functions."""
train_features, train_targets, train_atoms, test_features, \
test_targets, test_atoms = self.get_data()
# Test prediction routine with gaussian kernel.
kdict = {'k1': {'type': 'gaussian', 'width': 1., 'scaling': 1.}}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kdict,
regularization=1e-3,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True,
uncertainty=True)
print('gaussian prediction (rmse):',
pred['validation_error']['rmse_average'])
af = AcquisitionFunctions()
acq = af.rank(
targets=train_targets,
predictions=pred['prediction'],
uncertainty=pred['uncertainty'],
train_features=train_features,
test_features=test_features,
metrics=['cdf', 'optimistic', 'gaussian', 'UCB', 'EI', 'PI'])
self.assertTrue(len(acq['cdf']) == len(pred['prediction']))
self.assertTrue(len(acq['optimistic']) == len(pred['prediction']))
self.assertTrue(len(acq['gaussian']) == len(pred['prediction']))
self.assertTrue(len(acq['UCB']) == len(pred['prediction']))
self.assertTrue(len(acq['EI']) == len(pred['prediction']))
self.assertTrue(len(acq['PI']) == len(pred['prediction']))
acq = af.classify(
classifier,
train_atoms,
test_atoms,
targets=train_targets,
predictions=pred['prediction'],
uncertainty=pred['uncertainty'],
train_features=train_features,
test_features=test_features,
metrics=['cdf', 'optimistic', 'gaussian', 'UCB', 'EI', 'PI'])
self.assertTrue(len(acq['cdf']) == len(pred['prediction']))
self.assertTrue(len(acq['optimistic']) == len(pred['prediction']))
self.assertTrue(len(acq['gaussian']) == len(pred['prediction']))
self.assertTrue(len(acq['UCB']) == len(pred['prediction']))
self.assertTrue(len(acq['EI']) == len(pred['prediction']))
self.assertTrue(len(acq['PI']) == len(pred['prediction']))
def test_hyperparameter_opt(self):
kdict = [{'type': 'gaussian', 'width': width, 'scaling': scaling}]
gp_hyp = GaussianProcess(
kernel_list=kdict, train_fp=train, train_target=target,
gradients=gradients, scale_data=True,
optimize_hyperparameters=False, regularization=np.sqrt(1e-3))
gp_hyp.optimize_hyperparameters(global_opt=False)
bigKtilde_hyp = np.linalg.inv(gp_hyp.cinv)
bigKdg_hyp = bigKtilde_hyp[n_train:n_train + n_train * D, :n_train]
bigKgd_hyp = bigKtilde_hyp[:n_train, n_train:n_train + n_train * D]
print('Checking block matrices bigKgd and bigKdg (opt. hyper.).')
np.testing.assert_array_almost_equal(bigKdg_hyp, bigKgd_hyp.T,
decimal=10)
def train_model(self, train_features, train_targets):
"""Function to train a Gaussian process."""
kdict = [{
'type': 'gaussian',
'width': 0.5,
'scaling': 1.
}, {
'type': 'linear',
'scaling': 1.
}, {
'type': 'constant',
'const': 1.
}, {
'type': 'quadratic',
'slope': 1.,
'degree': 1.,
'scaling': 1.
}]
self.__class__.gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kdict,
regularization=np.sqrt(1e-3),
optimize_hyperparameters=True,
scale_data=False)
io.write(filename='test-model', model=self.gp, ext='pkl')
io.write(filename='test-model', model=self.gp, ext='hdf5')
def read(filename, ext='pkl'):
"""Function to read a pickle of model object.
Parameters
----------
filename : str
The name of the save file.
ext : str
Format to save GP, can be pkl or hdf5. Default is pkl.
Returns
-------
model : obj
Python GaussianProcess object.
"""
if ext is 'pkl':
with open('{}.pkl'.format(filename), 'rb') as infile:
return pickle.load(infile)
elif ext is 'hdf5':
train_features, train_targets, regularization, kernel_dict = \
read_train_data(filename)
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kernel_dict,
regularization=regularization,
optimize_hyperparameters=False)
return gp
else:
raise NotImplementedError(
'{} file extension not implemented.'.format(ext))
def minimize_error_time(train_features, train_targets, test_features,
test_targets):
"""A generic fitness function.
This fitness function will minimize the cost function as well as the time
to train the model. This will provide a Pareto optimial set of solutions
upon convergence.
Parameters
----------
train_features : array
The training features.
train_targets : array
The training targets.
test_features : array
The test feaatures.
test_targets : array
The test targets.
"""
kernel = {
'k1': {
'type': 'gaussian',
'width': 1.,
'scaling': 1.,
'dimension': 'single'
}
}
stime = time.time()
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kernel,
regularization=1e-2,
optimize_hyperparameters=True,
scale_data=True)
timing = time.time() - stime
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
score = pred['validation_error']['rmse_average']
return [-score, -timing]
def _train_model(train_features, train_targets):
kdict = {'k1': {'type': 'gaussian', 'width': 0.5}}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kdict,
regularization=1e-3,
optimize_hyperparameters=False,
scale_data=True)
return gp
def gp_predict(train_features, train_targets, test_features, test_targets):
"""Function to perform the GP predictions."""
data = {}
kdict = [
{'type': 'gaussian', 'width': 1., 'scaling': 1., 'dimension': 'single'},
]
gp = GaussianProcess(train_fp=train_features, train_target=train_targets,
kernel_list=kdict, regularization=1e-2,
optimize_hyperparameters=True, scale_data=True)
pred = gp.predict(test_fp=test_features)
data['result'] = get_error(pred['prediction'],
test_targets)['rmse_average']
data['size'] = len(train_targets)
return data
def minimize_error_descriptors(train_features, train_targets, test_features,
test_targets):
"""A generic fitness function.
This fitness function will minimize the cost function as well as the number
of descriptors. This will provide a Pareto optimial set of solutions upon
convergence.
Parameters
----------
train_features : array
The training features.
train_targets : array
The training targets.
test_features : array
The test feaatures.
test_targets : array
The test targets.
"""
kernel = [{
'type': 'gaussian',
'width': 1.,
'scaling': 1.,
'dimension': 'single'
}]
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kernel,
regularization=1e-2,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
score = pred['validation_error']['rmse_average']
dimension = train_features.shape[1]
return [-score, -dimension]
def fit(X, y, gradients, kdict):
""" Train the Gaussian process."""
gp = GaussianProcess(kernel_list=kdict,
regularization=0.0,
regularization_bounds=(0.0, 0.0),
train_fp=X,
train_target=y,
gradients=gradients,
optimize_hyperparameters=False)
print('Gaussiaon Process process trained.')
return gp
def minimize_error(train_features, train_targets, test_features, test_targets):
"""A generic fitness function.
This fitness function will minimize the cost function.
Parameters
----------
train_features : array
The training features.
train_targets : array
The training targets.
test_features : array
The test feaatures.
test_targets : array
The test targets.
"""
kernel = {
'k1': {
'type': 'gaussian',
'width': 1.,
'scaling': 1.,
'dimension': 'single'
}
}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kernel,
regularization=1e-2,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
score = pred['validation_error']['rmse_average']
return [-score]
def _get_opt_weights(self):
"""Function to get optimized kernel weights."""
# Train the GP.
gp = GaussianProcess(
train_fp=self.train_matrix, train_target=self.train_targets,
kernel_list=self.kernel_list, regularization=self.reg,
optimize_hyperparameters=True, scale_data=True
)
self.kernel_list = gp.kernel_list
self.reg = gp.regularization
return gp
def fitf(train_features, train_targets, test_features, test_targets):
"""Define the fitness function for the GA."""
kdict = [{
'type': 'gaussian',
'width': 1.,
'scaling': 1.,
'dimension': 'single'
}]
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kdict,
regularization=1e-2,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True)
score = pred['validation_error']['rmse_average']
return -score
def _surrogate_model(train_features,
train_targets,
test_features=None,
test_targets=None):
kdict = [{'type': 'gaussian', 'width': 0.5}]
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kdict,
regularization=1e-3,
optimize_hyperparameters=False,
scale_data=True)
if test_targets is None:
get_validation_error = False
else:
get_validation_error = True
score = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=get_validation_error,
get_training_error=False,
uncertainty=True)
to_acquire = np.argsort(
probability_density(0., score['prediction'], score['uncertainty']))
return to_acquire, score
def train_predict(train_features, train_targets):
"""Define the model."""
kdict = [{
'type': 'gaussian',
'width': 1.,
'scaling': 1.,
'dimension': 'single'
}]
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kdict,
regularization=np.sqrt(1e-2),
optimize_hyperparameters=True,
scale_data=True)
return gp
def test_gp_update(self):
"""Test Gaussian process predictions with the multiplication kernel."""
train_features, train_targets, test_features, test_targets = get_data()
kdict = {'k1': {'type': 'linear', 'scaling': 1.},
'k2': {'type': 'gaussian', 'width': 1., 'scaling': 1.,
'operation': 'multiplication'},
'c1': {'type': 'constant', 'const': 1.}}
gp = GaussianProcess(
train_fp=train_features, train_target=train_targets,
kernel_dict=kdict, regularization=1e-3,
optimize_hyperparameters=True, scale_data=True)
# Test updating the last model.
d, f = np.shape(train_features)
train_features = np.concatenate((train_features, test_features))
new_features = np.random.random_sample(
(np.shape(train_features)[0], 5))
train_features = np.concatenate((train_features, new_features), axis=1)
self.assertNotEqual(np.shape(train_features), (d, f))
train_targets = np.concatenate((train_targets, test_targets))
new_features = np.random.random_sample((len(test_features), 5))
test_features = np.concatenate((test_features, new_features), axis=1)
kdict = {'k1': {'type': 'linear', 'scaling': 1.},
'k2': {'type': 'gaussian', 'width': 1., 'scaling': 1.,
'operation': 'multiplication'},
'c1': {'type': 'constant', 'const': 1.}}
gp.update_gp(train_fp=train_features, train_target=train_targets,
kernel_dict=kdict)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
self.assertEqual(len(pred['prediction']), len(test_features))
print('Update prediction:',
pred['validation_error']['rmse_average'])
def test_gp_gaussian_kernel(self):
"""Test Gaussian process predictions with the gaussian kernel."""
train_features, train_targets, test_features, test_targets = get_data()
# Test prediction routine with gaussian kernel.
kdict = {'k1': {'type': 'gaussian', 'width': 1., 'scaling': 1.}}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kdict,
regularization=1e-3,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True,
uncertainty=True,
epsilon=0.1)
mult_dim = pred['prediction']
self.assertEqual(len(pred['prediction']), len(test_features))
print('gaussian prediction (rmse):',
pred['validation_error']['rmse_average'])
print('gaussian prediction (ins):',
pred['validation_error']['insensitive_average'])
print('gaussian prediction (abs):',
pred['validation_error']['absolute_average'])
# Test prediction routine with single width parameter.
kdict = {
'k1': {
'type': 'gaussian',
'width': 1.,
'scaling': 1.,
'dimension': 'single',
'bounds': ((1e-5, None), ),
'scaling_bounds': ((0., None), )
}
}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kdict,
regularization=1e-3,
optimize_hyperparameters=True,
scale_data=True)
self.assertEqual(len(gp.kernel_dict['k1']['width']), 1)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True,
uncertainty=True,
epsilon=0.1)
self.assertEqual(len(pred['prediction']), len(test_features))
self.assertNotEqual(np.sum(pred['prediction']), np.sum(mult_dim))
print('gaussian single width (rmse):',
pred['validation_error']['rmse_average'])
ax224 = fig.add_subplot(grid + last)
plt.title('Uncertainty Profiles')
plt.xlabel('Descriptor')
plt.ylabel('Uncertainty')
if True:
# Model example 1 - biased model.
# Define prediction parameters.
sdt1 = 0.001
# Too large width results in a biased model.
w1 = 3.0
kdict = {'k1': {'type': 'gaussian', 'width': w1}}
# Set up the prediction routine.
gp = GaussianProcess(kernel_dict=kdict,
regularization=sdt1**2,
train_fp=std['train'],
train_target=train_targets['target'],
optimize_hyperparameters=False)
# Do predictions.
under_fit = gp.predict(test_fp=std['test'], uncertainty=True)
# Scale predictions back to the original scale.
under_prediction = np.vstack(under_fit['prediction']) * \
train_targets['std'] + train_targets['mean']
under_uncertainty = np.vstack(under_fit['uncertainty']) * \
train_targets['std']
# Get average errors.
error = get_error(under_prediction.reshape(-1), afunc(test).reshape(-1))
print('Gaussian linear regression prediction:', error['absolute_average'])
# Get confidence interval on predictions.
upper = under_prediction + under_uncertainty * tstd
lower = under_prediction - under_uncertainty * tstd
class GeneralGaussianProcess(object):
"""Define a general setup for the Gaussin process.
This should not be used to try and obtain highly accurate solutions. Though
it should give a reasonable model.
"""
def __init__(self, clean_type='eliminate', dimension='single',
kernel='general'):
"""Initialize the class.
Parameters
----------
clean_type : str
Define method for handling missing data. Currently only elimination
implemented.
dimension : str
The number of parameters to return. Can be 'single', or 'all'.
kernel : str
Pass 'smooth' if a smooth but non-linear function is expected.
"""
self.clean_type = clean_type
self.dimension = dimension
self.kernel = kernel
def train_gaussian_process(self, train_features, train_targets):
"""Generate a general gaussian process model.
Parameters
----------
train_features : array
The array of training features.
train_targets : array
A list of training target values.
Returns
-------
gp : object
The trained Gaussian process.
"""
train_features, train_targets = self._process_train_data(
train_features, train_targets)
if 'smooth' in self.kernel:
kdict = smooth_kernel(train_features, self.dimension)
else:
kdict = general_kernel(train_features, self.dimension)
self.gp = GaussianProcess(
train_fp=train_features, train_target=train_targets,
kernel_list=kdict, regularization=1e-2,
optimize_hyperparameters=True, scale_data=False
)
return self.gp
def gaussian_process_predict(self, test_features):
"""Function to make GP predictions on tests data.
Parameters
----------
test_features : array
The array of test features.
Returns
-------
prediction : dict
The prediction data generated by the Gaussian process.
"""
test_features = self.cleaner.transform(test_features)
pred = self.gp.predict(test_fp=test_features)
return pred
def _process_train_data(self, train_features, train_targets):
"""Prepare the data.
Parameters
----------
train_features : array
The array of training features.
train_targets : array
A list of training target values.
Returns
-------
train_features : array
The array of cleaned and scaled training features.
train_targets : array
A list of cleaned training target values.
"""
self.cleaner = GeneralPrepreprocess(clean_type=self.clean_type)
train_features, train_targets, _ = self.cleaner.process(
train_features, train_targets)
return train_features, train_targets
# ## Baseline
#
# Initially, a GP is trained on all the features and the error calculated as a baseline.
# In[5]:
kdict = [{
'type': 'gaussian',
'width': 1.,
'scaling': 1.,
'dimension': 'single'
}]
gp = GaussianProcess(train_fp=train_data,
train_target=trainval,
kernel_list=kdict,
regularization=1e-2,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_data,
test_target=testval,
get_validation_error=True,
get_training_error=True)
score = pred['validation_error']['rmse_average']
print('all features: {0:.3f}'.format(score))
# ## Optimization
#
# To optimize the feature set, a prediction function is first defined. This will take a boolean array from the genetic algorithm, transform the feature set and test against the holdout data. The error is then calculated and returned for that set of features. The genetic algorithm will aim to maximize the "fitness" of a population of candidates; therefore, the negative of the average cost is returned in this example.
# The `tstd` variable specifies how many standard deviations we plot.
#
# # Model example 1 - Gaussian linear kernel regression.
# In[4]:
# Define prediction parameters
kdict = [{'type': 'linear', 'scaling': 1.},
{'type': 'constant', 'const': 0.}]
# Starting guess for the noise parameter
sdt1 = noise_magnitude
# Set up the gaussian process.
gp1 = GaussianProcess(kernel_list=kdict, regularization=np.sqrt(sdt1),
train_fp=std['train'],
train_target=train_targets['target'],
optimize_hyperparameters=True)
# Do predictions.
linear = gp1.predict(test_fp=std['test'], uncertainty=True)
# Put predictions back on real scale.
prediction = np.vstack(linear['prediction']) * train_targets['std'] + train_targets['mean']
# Put uncertainties back on real scale.
uncertainty = np.vstack(linear['uncertainty_with_reg']) * train_targets['std']
# Get confidence interval on predictions.
over_upper = prediction + uncertainty * tstd
over_lower = prediction - uncertainty * tstd
# Plotting.
plt3d = plt.figure(0).gca(projection='3d')
# Plot training data.
plt3d.scatter(train[:, 0], train[:, 1], target, color='b')
# ## Scaling parameter <a name="constant-parameter"></a>
# [(Back to top)](#head)
#
# The scaling parameter $\sigma_v$ is varied in the following. We choose a range of values in the range of `[1., 1e-2, 1e-4, 1e-6]`, iterate over them and plot the predictions from the resulting model. It is seen that as the scaling parameter is decreased, the slope on the predictions also decreases. For the smallest scaling parameter of `1e-6`, there is essentially zero slope and the Gaussian process simply predicts the mean of the data.
# In[5]:
fig = plt.figure(figsize=(20, 10))
for w, p in zip([1., 1e-2, 1e-4, 1e-6], [141, 142, 143, 144]):
kdict = [{'type': 'linear', 'scaling': w}]
# Set up the prediction routine.
gp = GaussianProcess(kernel_list=kdict, regularization=np.sqrt(1e-3),
train_fp=train,
train_target=target,
optimize_hyperparameters=False, scale_data=True)
# Do predictions.
fit = gp.predict(test_fp=test, uncertainty=True)
# Get average errors.
error = get_error(fit['prediction'], afunc(test))
print('Gaussian regression error with {0} width: {1:.3f}'.format(
w, error['absolute_average']))
# Plotting.
plot(p, fit['prediction'])
# ## Regularization parameter <a name="constant-parameter"></a>
# [(Back to top)](#head)
