def test_gp_gaussian_kernel(self):
"""Test Gaussian process predictions with the gaussian kernel."""
train_features, train_targets, test_features, test_targets = get_data()
# Test prediction routine with gaussian kernel.
kdict = {'k1': {'type': 'gaussian', 'width': 1., 'scaling': 1.}}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kdict,
regularization=1e-3,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True,
uncertainty=True,
epsilon=0.1)
mult_dim = pred['prediction']
self.assertEqual(len(pred['prediction']), len(test_features))
print('gaussian prediction (rmse):',
pred['validation_error']['rmse_average'])
print('gaussian prediction (ins):',
pred['validation_error']['insensitive_average'])
print('gaussian prediction (abs):',
pred['validation_error']['absolute_average'])
# Test prediction routine with single width parameter.
kdict = {
'k1': {
'type': 'gaussian',
'width': 1.,
'scaling': 1.,
'dimension': 'single',
'bounds': ((1e-5, None), ),
'scaling_bounds': ((0., None), )
}
}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kdict,
regularization=1e-3,
optimize_hyperparameters=True,
scale_data=True)
self.assertEqual(len(gp.kernel_dict['k1']['width']), 1)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True,
uncertainty=True,
epsilon=0.1)
self.assertEqual(len(pred['prediction']), len(test_features))
self.assertNotEqual(np.sum(pred['prediction']), np.sum(mult_dim))
print('gaussian single width (rmse):',
pred['validation_error']['rmse_average'])
def test_gp_addative_kernel(self):
"""Test Gaussian process predictions with the addative kernel."""
train_features, train_targets, test_features, test_targets = get_data()
# Test prediction with addative linear and gaussian kernel.
kdict = {
'k1': {
'type': 'linear',
'features': [0, 1],
'scaling': 1.
},
'k2': {
'type': 'gaussian',
'features': [2, 3],
'width': 1.,
'scaling': 1.
},
'c1': {
'type': 'constant',
'const': 1.
}
}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kdict,
regularization=1e-3,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
self.assertEqual(len(pred['prediction']), len(test_features))
print('addition prediction:', pred['validation_error']['rmse_average'])
def test_gp_laplacian_kernel(self):
"""Test Gaussian process predictions with the laplacian kernel."""
train_features, train_targets, test_features, test_targets = get_data()
# Test prediction routine with laplacian kernel.
kdict = [{
'type': 'laplacian',
'width': 1.,
'scaling': 1.,
'bounds': ((1e-5, None), ) * np.shape(train_features)[1],
'scaling_bounds': ((0., None), )
}]
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kdict,
regularization=np.sqrt(1e-3),
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
self.assertEqual(len(pred['prediction']), len(test_features))
print('laplacian prediction:',
pred['validation_error']['rmse_average'])
def test_gp_multiplication_kernel(self):
"""Test Gaussian process predictions with the multiplication kernel."""
train_features, train_targets, test_features, test_targets = get_data()
# Test prediction with multiplication of linear & gaussian kernel.
kdict = [{
'type': 'linear',
'features': [0, 1],
'scaling': 1.
}, {
'type': 'gaussian',
'features': [2, 3],
'width': 1.,
'scaling': 1.,
'operation': 'multiplication'
}, {
'type': 'constant',
'const': 1.
}]
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kdict,
regularization=np.sqrt(1e-3),
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
self.assertEqual(len(pred['prediction']), len(test_features))
print('multiplication prediction:',
pred['validation_error']['rmse_average'])
def test_gp(self):
"""Test Gaussian process predictions with rescaling."""
train_features, train_targets, test_features, test_targets = get_data()
# Test prediction routine with gaussian kernel.
kdict = {'k1': {'type': 'gaussian', 'width': 1., 'scaling': 1.}}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kdict,
regularization=1e-3,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True,
uncertainty=True)
opt = gp.kernel_dict['k1']['width']
orig = hs.rescale_hyperparameters(gp.scaling,
gp.kernel_dict)['k1']['width']
assert not np.allclose(opt, orig)
scaled = hs.hyperparameters(gp.scaling, gp.kernel_dict)['k1']['width']
assert np.allclose(opt, scaled)
print('gaussian prediction (rmse):',
pred['validation_error']['rmse_average'])
def test_gp_quadratic_kernel(self):
"""Test Gaussian process predictions with the quadratic kernel."""
train_features, train_targets, test_features, test_targets = get_data()
# Test prediction routine with quadratic kernel.
kdict = {
'k1': {
'type': 'quadratic',
'slope': 1.,
'degree': 1.,
'scaling': 1.,
'bounds': ((1e-5, None), ) * (np.shape(train_features)[1] + 1),
'scaling_bounds': ((0., None), )
}
}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kdict,
regularization=1e-3,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
self.assertEqual(len(pred['prediction']), len(test_features))
print('quadratic prediction:',
pred['validation_error']['rmse_average'])
def test_gp_linear_kernel(self):
"""Test Gaussian process predictions with the linear kernel."""
train_features, train_targets, test_features, test_targets = get_data()
# Test prediction routine with linear kernel.
kdict = {
'k1': {
'type': 'linear',
'scaling': 1.,
'scaling_bounds': ((0., None), )
},
'c1': {
'type': 'constant',
'const': 1.
}
}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kdict,
regularization=1e-3,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
self.assertEqual(len(pred['prediction']), len(test_features))
print('linear prediction:', pred['validation_error']['rmse_average'])
def prediction(self, train_features, train_targets, test_features,
test_targets):
"""Ridge regression predictions."""
# Test ridge regression predictions.
sigma = 1.
kdict = [{'type': 'gaussian', 'width': sigma, 'dimension': 'single'}]
regularization = np.sqrt(0.001)
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kdict,
regularization=regularization,
optimize_hyperparameters=False,
scale_data=True)
output = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True,
uncertainty=True)
r = [
np.shape(train_features)[0],
output['validation_error']['rmse_average'],
output['validation_error']['absolute_average'],
np.mean(output['uncertainty'])
]
return r
def test_gp_update(self):
"""Test Gaussian process predictions with the multiplication kernel."""
train_features, train_targets, test_features, test_targets = get_data()
kdict = [{
'type': 'linear',
'scaling': 1.
}, {
'type': 'gaussian',
'width': 1.,
'scaling': 1.,
'operation': 'multiplication'
}, {
'type': 'constant',
'const': 1.
}]
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kdict,
regularization=np.sqrt(1e-3),
optimize_hyperparameters=True,
scale_data=True)
# Test updating the last model.
d, f = np.shape(train_features)
train_features = np.concatenate((train_features, test_features))
new_features = np.random.random_sample(
(np.shape(train_features)[0], 5))
train_features = np.concatenate((train_features, new_features), axis=1)
self.assertNotEqual(np.shape(train_features), (d, f))
train_targets = np.concatenate((train_targets, test_targets))
new_features = np.random.random_sample((len(test_features), 5))
test_features = np.concatenate((test_features, new_features), axis=1)
kdict = [{
'type': 'linear',
'scaling': 1.
}, {
'type': 'gaussian',
'width': 1.,
'scaling': 1.,
'operation': 'multiplication'
}, {
'type': 'constant',
'const': 1.
}]
gp.update_gp(train_fp=train_features,
train_target=train_targets,
kernel_list=kdict)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
sigma = gp.predict_uncertainty(test_fp=test_features)
self.assertEqual(len(sigma['uncertainty']), len(test_features))
self.assertEqual(len(pred['prediction']), len(test_features))
print('Update prediction:', pred['validation_error']['rmse_average'])
def test_acquisition(self):
"""Test acquisition functions."""
train_features, train_targets, train_atoms, test_features, \
test_targets, test_atoms = self.get_data()
# Test prediction routine with gaussian kernel.
kdict = {'k1': {'type': 'gaussian', 'width': 1., 'scaling': 1.}}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kdict,
regularization=1e-3,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True,
uncertainty=True)
print('gaussian prediction (rmse):',
pred['validation_error']['rmse_average'])
af = AcquisitionFunctions()
acq = af.rank(
targets=train_targets,
predictions=pred['prediction'],
uncertainty=pred['uncertainty'],
train_features=train_features,
test_features=test_features,
metrics=['cdf', 'optimistic', 'gaussian', 'UCB', 'EI', 'PI'])
self.assertTrue(len(acq['cdf']) == len(pred['prediction']))
self.assertTrue(len(acq['optimistic']) == len(pred['prediction']))
self.assertTrue(len(acq['gaussian']) == len(pred['prediction']))
self.assertTrue(len(acq['UCB']) == len(pred['prediction']))
self.assertTrue(len(acq['EI']) == len(pred['prediction']))
self.assertTrue(len(acq['PI']) == len(pred['prediction']))
acq = af.classify(
classifier,
train_atoms,
test_atoms,
targets=train_targets,
predictions=pred['prediction'],
uncertainty=pred['uncertainty'],
train_features=train_features,
test_features=test_features,
metrics=['cdf', 'optimistic', 'gaussian', 'UCB', 'EI', 'PI'])
self.assertTrue(len(acq['cdf']) == len(pred['prediction']))
self.assertTrue(len(acq['optimistic']) == len(pred['prediction']))
self.assertTrue(len(acq['gaussian']) == len(pred['prediction']))
self.assertTrue(len(acq['UCB']) == len(pred['prediction']))
self.assertTrue(len(acq['EI']) == len(pred['prediction']))
self.assertTrue(len(acq['PI']) == len(pred['prediction']))
def minimize_error_time(train_features, train_targets, test_features,
test_targets):
"""A generic fitness function.
This fitness function will minimize the cost function as well as the time
to train the model. This will provide a Pareto optimial set of solutions
upon convergence.
Parameters
----------
train_features : array
The training features.
train_targets : array
The training targets.
test_features : array
The test feaatures.
test_targets : array
The test targets.
"""
kernel = {
'k1': {
'type': 'gaussian',
'width': 1.,
'scaling': 1.,
'dimension': 'single'
}
}
stime = time.time()
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kernel,
regularization=1e-2,
optimize_hyperparameters=True,
scale_data=True)
timing = time.time() - stime
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
score = pred['validation_error']['rmse_average']
return [-score, -timing]
def gp_predict(train_features, train_targets, test_features, test_targets):
"""Function to perform the GP predictions."""
data = {}
kdict = [
{'type': 'gaussian', 'width': 1., 'scaling': 1., 'dimension': 'single'},
]
gp = GaussianProcess(train_fp=train_features, train_target=train_targets,
kernel_list=kdict, regularization=1e-2,
optimize_hyperparameters=True, scale_data=True)
pred = gp.predict(test_fp=test_features)
data['result'] = get_error(pred['prediction'],
test_targets)['rmse_average']
data['size'] = len(train_targets)
return data
def minimize_error_descriptors(train_features, train_targets, test_features,
test_targets):
"""A generic fitness function.
This fitness function will minimize the cost function as well as the number
of descriptors. This will provide a Pareto optimial set of solutions upon
convergence.
Parameters
----------
train_features : array
The training features.
train_targets : array
The training targets.
test_features : array
The test feaatures.
test_targets : array
The test targets.
"""
kernel = [{
'type': 'gaussian',
'width': 1.,
'scaling': 1.,
'dimension': 'single'
}]
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kernel,
regularization=1e-2,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
score = pred['validation_error']['rmse_average']
dimension = train_features.shape[1]
return [-score, -dimension]
def minimize_error(train_features, train_targets, test_features, test_targets):
"""A generic fitness function.
This fitness function will minimize the cost function.
Parameters
----------
train_features : array
The training features.
train_targets : array
The training targets.
test_features : array
The test feaatures.
test_targets : array
The test targets.
"""
kernel = {
'k1': {
'type': 'gaussian',
'width': 1.,
'scaling': 1.,
'dimension': 'single'
}
}
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_dict=kernel,
regularization=1e-2,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True,
get_training_error=True)
score = pred['validation_error']['rmse_average']
return [-score]
def fitf(train_features, train_targets, test_features, test_targets):
"""Define the fitness function for the GA."""
kdict = [{
'type': 'gaussian',
'width': 1.,
'scaling': 1.,
'dimension': 'single'
}]
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kdict,
regularization=1e-2,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=True)
score = pred['validation_error']['rmse_average']
return -score
def _surrogate_model(train_features,
train_targets,
test_features=None,
test_targets=None):
kdict = [{'type': 'gaussian', 'width': 0.5}]
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kdict,
regularization=1e-3,
optimize_hyperparameters=False,
scale_data=True)
if test_targets is None:
get_validation_error = False
else:
get_validation_error = True
score = gp.predict(test_fp=test_features,
test_target=test_targets,
get_validation_error=get_validation_error,
get_training_error=False,
uncertainty=True)
to_acquire = np.argsort(
probability_density(0., score['prediction'], score['uncertainty']))
return to_acquire, score
#
# Example 1: Biased model
#
sdt1 = 0.001
w1 = 3.0 # too large width results in a biased model
kdict = [{"type": "gaussian", "width": w1}]
gp = GaussianProcess(kernel_list=kdict,
regularization=sdt1,
train_fp=std["train"],
train_target=train_targets["target"],
optimize_hyperparameters=False)
under_fit = gp.predict(test_fp=std["test"], uncertainty=True)
# Scale predictions back to the original scale
under_prediction = np.vstack(
under_fit["prediction"]) * train_targets["std"] + train_targets["mean"]
under_certainty = np.vstack(
under_fit["uncertainty_with_reg"]) * train_targets["std"]
# Get average errors
error = get_error(under_prediction.reshape(-1), my_func(test).reshape(-1))
# Get confidence interval on predictions
upper = under_prediction + under_certainty * tstd
lower = under_prediction - under_certainty * tstd
plt.figure(0)
# In[5]:
fig = plt.figure(figsize=(20, 10))
for w, p in zip([1.5, 1., 0.5, 0.1], [141, 142, 143, 144]):
kdict = [{'type': 'gaussian', 'width': w, 'scaling': 1.}]
# Set up the prediction routine.
gp = GaussianProcess(kernel_list=kdict,
regularization=np.sqrt(1e-3),
train_fp=train,
train_target=target,
optimize_hyperparameters=False,
scale_data=True)
# Do predictions.
fit = gp.predict(test_fp=test)
# Get average errors.
error = get_error(fit['prediction'], afunc(test))
print('Gaussian regression error with {0} width: {1:.3f}'.format(
w, error['absolute_average']))
# Plotting.
plot(p, fit['prediction'])
# ## Scaling Parameter <a name="scaling-parameter"></a>
# [(Back to top)](#head)
#
# And then the same study can be performed with the scaling parameter ($\sigma$).
# In[6]:
# In[4]:
# Define prediction parameters
kdict = [{'type': 'linear', 'scaling': 1.},
{'type': 'constant', 'const': 0.}]
# Starting guess for the noise parameter
sdt1 = noise_magnitude
# Set up the gaussian process.
gp1 = GaussianProcess(kernel_list=kdict, regularization=np.sqrt(sdt1),
train_fp=std['train'],
train_target=train_targets['target'],
optimize_hyperparameters=True)
# Do predictions.
linear = gp1.predict(test_fp=std['test'], uncertainty=True)
# Put predictions back on real scale.
prediction = np.vstack(linear['prediction']) * train_targets['std'] + train_targets['mean']
# Put uncertainties back on real scale.
uncertainty = np.vstack(linear['uncertainty_with_reg']) * train_targets['std']
# Get confidence interval on predictions.
over_upper = prediction + uncertainty * tstd
over_lower = prediction - uncertainty * tstd
# Plotting.
plt3d = plt.figure(0).gca(projection='3d')
# Plot training data.
plt3d.scatter(train[:, 0], train[:, 1], target, color='b')
# Plot exact function.
plt3d.plot_surface(test_x1, test_x2,
'scaling': 1.,
'dimension': 'single'
},
{
'type': 'linear',
'scaling': 1.
},
]
gp = GaussianProcess(train_fp=train_features,
train_target=train_targets,
kernel_list=kdict,
regularization=1e-2,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_features)
error = get_error(pred['prediction'], test_targets)['rmse_average']
print(error)
plt.figure(6)
plt.figure(figsize=(30, 15))
plt.plot(test_targets, pred['prediction'], 'o', c='r', alpha=0.5)
plt.savefig('gaussian_process.png')
# Here we see that the Gaussian process performs slightly worse than the simple ridge regression model. This is to be expected when we are trying to model linear data with a non-linear model. However, the inclusion of the linear kernel results in a good prediction error. If the squared exponential kernel were to be removed from the above example, the resulting model would be the same as the ridge regression model, just trained with the Gaussian process.
#
# ## Cross-validation <a name="cross-validation"></a>
# [(Back to top)](#head)
#
},
'c1': {
'type': 'constant',
'const': 0.
}
}
# Starting guess for the noise parameter
sdt1 = noise_magnitude
# Set up the gaussian process.
gp1 = GaussianProcess(kernel_dict=kdict,
regularization=sdt1,
train_fp=std['train'],
train_target=train_targets['target'],
optimize_hyperparameters=True)
# Do predictions.
linear = gp1.predict(test_fp=std['test'], uncertainty=True)
# Put predictions back on real scale.
prediction = np.vstack(linear['prediction']) * train_targets['std'] + \
train_targets['mean']
# Put uncertainties back on real scale.
uncertainty = np.vstack(linear['uncertainty']) * train_targets['std']
# Get confidence interval on predictions.
over_upper = prediction + uncertainty * tstd
over_lower = prediction - uncertainty * tstd
# Plot the uncertainties upper and lower bounds.
plt3d.plot_surface(test_x1,
test_x2,
over_upper.reshape(np.shape(test_x1)),
alpha=0.3,
color='r')
plt3d.plot_surface(test_x1,
# The scaling parameter $\sigma_v$ is varied in the following. We choose a range of values in the range of `[1., 1e-2, 1e-4, 1e-6]`, iterate over them and plot the predictions from the resulting model. It is seen that as the scaling parameter is decreased, the slope on the predictions also decreases. For the smallest scaling parameter of `1e-6`, there is essentially zero slope and the Gaussian process simply predicts the mean of the data.
# In[5]:
fig = plt.figure(figsize=(20, 10))
for w, p in zip([1., 1e-2, 1e-4, 1e-6], [141, 142, 143, 144]):
kdict = [{'type': 'linear', 'scaling': w}]
# Set up the prediction routine.
gp = GaussianProcess(kernel_list=kdict, regularization=np.sqrt(1e-3),
train_fp=train,
train_target=target,
optimize_hyperparameters=False, scale_data=True)
# Do predictions.
fit = gp.predict(test_fp=test, uncertainty=True)
# Get average errors.
error = get_error(fit['prediction'], afunc(test))
print('Gaussian regression error with {0} width: {1:.3f}'.format(
w, error['absolute_average']))
# Plotting.
plot(p, fit['prediction'])
# ## Regularization parameter <a name="constant-parameter"></a>
# [(Back to top)](#head)
#
# The regularization parameter is varied within the range of `[1., 1e-1, 1e-2, 1e-3]`. Here we find that for larger values the model will under-fit. This will essentially result in the mean of the data being returned across the range of test values. When the regularization parameter gets small enough, it will have little impact on the model predictions as it will be smaller than the noise on the data.
kdict = [{
'type': 'gaussian',
'width': 1.,
'scaling': 1.,
'dimension': 'single'
}]
gp = GaussianProcess(train_fp=train_data,
train_target=trainval,
kernel_list=kdict,
regularization=1e-2,
optimize_hyperparameters=True,
scale_data=True)
pred = gp.predict(test_fp=test_data,
test_target=testval,
get_validation_error=True,
get_training_error=True)
score = pred['validation_error']['rmse_average']
print('all features: {0:.3f}'.format(score))
# ## Optimization
#
# To optimize the feature set, a prediction function is first defined. This will take a boolean array from the genetic algorithm, transform the feature set and test against the holdout data. The error is then calculated and returned for that set of features. The genetic algorithm will aim to maximize the "fitness" of a population of candidates; therefore, the negative of the average cost is returned in this example.
# In[6]:
def fitf(train_features, train_targets, test_features, test_targets):
"""Define the fitness function for the GA."""
class GeneralGaussianProcess(object):
"""Define a general setup for the Gaussin process.
This should not be used to try and obtain highly accurate solutions. Though
it should give a reasonable model.
"""
def __init__(self, clean_type='eliminate', dimension='single',
kernel='general'):
"""Initialize the class.
Parameters
----------
clean_type : str
Define method for handling missing data. Currently only elimination
implemented.
dimension : str
The number of parameters to return. Can be 'single', or 'all'.
kernel : str
Pass 'smooth' if a smooth but non-linear function is expected.
"""
self.clean_type = clean_type
self.dimension = dimension
self.kernel = kernel
def train_gaussian_process(self, train_features, train_targets):
"""Generate a general gaussian process model.
Parameters
----------
train_features : array
The array of training features.
train_targets : array
A list of training target values.
Returns
-------
gp : object
The trained Gaussian process.
"""
train_features, train_targets = self._process_train_data(
train_features, train_targets)
if 'smooth' in self.kernel:
kdict = smooth_kernel(train_features, self.dimension)
else:
kdict = general_kernel(train_features, self.dimension)
self.gp = GaussianProcess(
train_fp=train_features, train_target=train_targets,
kernel_list=kdict, regularization=1e-2,
optimize_hyperparameters=True, scale_data=False
)
return self.gp
def gaussian_process_predict(self, test_features):
"""Function to make GP predictions on tests data.
Parameters
----------
test_features : array
The array of test features.
Returns
-------
prediction : dict
The prediction data generated by the Gaussian process.
"""
test_features = self.cleaner.transform(test_features)
pred = self.gp.predict(test_fp=test_features)
return pred
def _process_train_data(self, train_features, train_targets):
"""Prepare the data.
Parameters
----------
train_features : array
The array of training features.
train_targets : array
A list of training target values.
Returns
-------
train_features : array
The array of cleaned and scaled training features.
train_targets : array
A list of cleaned training target values.
"""
self.cleaner = GeneralPrepreprocess(clean_type=self.clean_type)
train_features, train_targets, _ = self.cleaner.process(
train_features, train_targets)
return train_features, train_targets
'type': 'constant',
'const': 0.0
}]
# Starting guess for the noise parameter
sdt1 = noise_magnitude
# Set up the gaussian process.
gp1 = GaussianProcess(kernel_list=kdict,
regularization=sdt1,
train_fp=std['train'],
train_target=train_targets['target'],
optimize_hyperparameters=True,
scale_optimizer=False)
# Do predictions.
linear = gp1.predict(test_fp=std['test'],
get_validation_error=True,
test_target=afunc(test))
prediction = np.array(
linear['prediction']) * train_targets['std'] + train_targets['mean']
error = get_error(prediction, afunc(test))
# Plotting.
plt3d = plt.figure(1).gca(projection='3d')
# Plot training data.
plt3d.scatter(train[:, 0], train[:, 1], target, color='b')
# Plot exact function.
plt3d.plot_surface(test_x1,
test_x2,
afunc(test).reshape(np.shape(test_x1)),
alpha=0.3,
color='b')
# Plot the prediction.
if True:
# Model example 1 - biased model.
# Define prediction parameters.
sdt1 = 0.001
# Too large width results in a biased model.
w1 = 3.0
kdict = {'k1': {'type': 'gaussian', 'width': w1}}
# Set up the prediction routine.
gp = GaussianProcess(kernel_dict=kdict,
regularization=sdt1**2,
train_fp=std['train'],
train_target=train_targets['target'],
optimize_hyperparameters=False)
# Do predictions.
under_fit = gp.predict(test_fp=std['test'], uncertainty=True)
# Scale predictions back to the original scale.
under_prediction = np.vstack(under_fit['prediction']) * \
train_targets['std'] + train_targets['mean']
under_uncertainty = np.vstack(under_fit['uncertainty']) * \
train_targets['std']
# Get average errors.
error = get_error(under_prediction.reshape(-1), afunc(test).reshape(-1))
print('Gaussian linear regression prediction:', error['absolute_average'])
# Get confidence interval on predictions.
upper = under_prediction + under_uncertainty * tstd
lower = under_prediction - under_uncertainty * tstd
# Plot example 1
ax = fig.add_subplot(grid + 1)
ax.plot(linex, liney, '-', lw=1, color='black')
class MLMin(object):
def __init__(self, x0, prev_calculations=None, restart=False,
trajectory='catlearn_opt.traj'):
"""Optimization setup.
Parameters
----------
x0 : Atoms object or trajectory file in ASE format.
Initial guess.
restart: boolean
Restart calculation. Only if a trajectory file with the same
name as the one introduced in the 'trajectory' flag is found.
This file must be in the same directory than the direction in
which the optimization was initiated.
prev_calculations: list of Atoms or trajectory file (ASE format).
The user can feed previously calculated data for the same
hypersurface (i.e. the calculations must be done using the same
calculator and parameters).
trajectory: string
Filename to store the output.
"""
# General variables.
self.prev_calculations = prev_calculations
self.filename = trajectory
self.iter = 0
self.fmax = 0.
self.gp = None
self.opt_type = 'MLMin'
self.version = self.opt_type + ' ' + __version__
print_version(self.version)
if restart is True and prev_calculations is None:
if os.path.isfile(self.filename):
self.prev_calculations = self.filename
msg = 'You must set an initial Atoms structure.'
assert x0 is not None, msg
# Read trajectory file.
if isinstance(x0, str):
x0 = read(x0, '-1')
self.ase_ini = x0
self.ase_calc = self.ase_ini.get_calculator()
msg = 'ASE calculator not found.'
assert self.ase_calc, msg
self.constraints = self.ase_ini.constraints
self.x0 = self.ase_ini.get_positions().flatten()
self.num_atoms = self.ase_ini.get_number_of_atoms()
# Information from the initial structure.
new_atoms = self.ase_ini
self.list_atoms = [new_atoms]
if self.prev_calculations is not None:
if isinstance(self.prev_calculations, str):
print('Reading previous calculations from a traj. file.')
self.prev_calculations = read(self.prev_calculations, ':')
# Check for duplicates.
for prev_atoms in self.prev_calculations:
duplicate = False
prev_pos = prev_atoms.get_positions().reshape(-1)
for atoms in self.list_atoms:
pos_atoms = atoms.get_positions().reshape(-1)
d_ij = euclidean(pos_atoms, prev_pos)
if d_ij == 0:
duplicate = True
if duplicate is False:
self.list_atoms += [prev_atoms]
# Extract information from previous calculations.
self.list_train = []
self.list_targets = []
self.list_gradients = []
for i in range(0, len(self.list_atoms)):
pos_atoms = list(self.list_atoms[i].get_positions().reshape(-1))
energy_atoms = self.list_atoms[i].get_potential_energy()
grad_atoms = list(-self.list_atoms[i].get_forces().reshape(-1))
self.list_train.append(pos_atoms)
self.list_targets.append(energy_atoms)
self.list_gradients.append(grad_atoms)
self.list_max_abs_forces = []
for i in self.list_gradients:
self.list_fmax = get_fmax(np.array([i]))
self.max_abs_forces = np.max(np.abs(self.list_fmax))
self.list_max_abs_forces.append(self.max_abs_forces)
# Get constraints.
self.index_mask = create_mask(self.ase_ini, self.constraints)
# Dump all Atoms to file.
write(self.filename, self.list_atoms)
print_info(self)
print(self.list_targets)
def run(self, fmax=0.05, steps=200, kernel='SQE', max_step=0.25,
acq='min_energy', full_output=False):
"""Executing run will start the optimization process.
Parameters
----------
fmax : float
Convergence criteria (in eV/Angstrom).
steps : int
Max. number of optimization steps.
kernel: string
Type of covariance function to be used.
Implemented are: SQE (fixed hyperparamters), SQE_opt and ARD_SQE.
max_step: float
Early stopping criteria. Maximum uncertainty before stopping the
optimization in the predicted landscape.
acq : string
The acquisition function that decides the next point to
evaluate. Implemented are: 'lcb', 'ucb', 'min_energy'.
full_output: boolean
Whether to print on screen the full output (True) or not (False).
Returns
-------
Optimized atom structure.
"""
self.fmax = fmax
self.acq = acq
success_hyper = False
while not converged(self):
# 1. Train Machine Learning model.
train = np.copy(self.list_train)
targets = np.copy(self.list_targets)
gradients = np.copy(self.list_gradients)
self.u_prior = np.max(targets)
scaled_targets = targets - self.u_prior
sigma_f = 1e-3 + np.std(scaled_targets)**2
if kernel == 'SQE_fixed':
opt_hyper = False
kdict = [{'type': 'gaussian', 'width': 0.4,
'dimension': 'single',
'bounds': ((0.4, 0.4),),
'scaling': 1.0,
'scaling_bounds': ((1.0, 1.0),)},
{'type': 'noise_multi',
'hyperparameters': [0.005 * 0.4**2, 0.005],
'bounds': ((0.005 * 0.4**2, 0.005 * 0.4**2),
(0.005, 0.005),)}
]
if kernel == 'SQE':
opt_hyper = True
kdict = [{'type': 'gaussian', 'width': 0.4,
'dimension': 'single',
'bounds': ((0.01, 1.0),),
'scaling': sigma_f,
'scaling_bounds': ((sigma_f, sigma_f),)},
{'type': 'noise_multi',
'hyperparameters': [0.005, 0.005 * 0.4**2],
'bounds': ((0.001, 0.050),
(0.001 * 0.4**2, 0.050),)}
]
if kernel == 'ARD_SQE':
opt_hyper = True
kdict = [{'type': 'gaussian', 'width': 0.4,
'dimension': 'features',
'bounds': ((0.01, 1.0),) * len(self.index_mask),
'scaling': sigma_f,
'scaling_bounds': ((sigma_f, sigma_f),)},
{'type': 'noise_multi',
'hyperparameters': [0.005, 0.0005],
'bounds': ((0.001, 0.005),
(0.0005, 0.002),)}
]
if success_hyper is not False:
kdict = success_hyper
if self.index_mask is not None:
train = apply_mask(list_to_mask=train,
mask_index=self.index_mask)[1]
gradients = apply_mask(list_to_mask=gradients,
mask_index=self.index_mask)[1]
print('Training a GP process...')
print('Number of training points:', len(scaled_targets))
train = train.tolist()
gradients = gradients.tolist()
self.gp = GaussianProcess(kernel_list=kdict,
regularization=0.0,
regularization_bounds=(0.0, 0.0),
train_fp=train,
train_target=scaled_targets,
gradients=gradients,
optimize_hyperparameters=False)
print('GP process trained.')
if opt_hyper is True:
if len(self.list_targets) > 5:
self.gp.optimize_hyperparameters()
if self.gp.theta_opt.success is True:
if full_output is True:
print('Hyperparam. optimization was successful.')
print('Updating kernel list...')
success_hyper = self.gp.kernel_list
if self.gp.theta_opt.success is not True:
if full_output is True:
print('Hyperparam. optimization unsuccessful.')
if success_hyper is False:
if full_output is True:
print('Not enough data...')
if success_hyper is not False:
if full_output is True:
print('Using the last optimized '
'hyperparamters.')
if full_output is True:
print('Kernel list:', self.gp.kernel_list)
# 2. Optimize Machine Learning model.
self.list_interesting_points = []
self.list_interesting_energies = []
self.list_interesting_uncertainties = []
guess = self.ase_ini
guess_pos = np.array(self.list_train[-1])
guess.positions = guess_pos.reshape(-1, 3)
guess.set_calculator(ASECalc(gp=self.gp,
index_constraints=self.index_mask,
scaling_targets=self.u_prior)
)
# Optimization in the predicted landscape:
ml_opt = MDMin(guess, trajectory=None, logfile=None, dt=0.020)
if full_output is True:
print('Starting optimization on the predicted landscape...')
ml_converged = False
n_steps_performed = 0
while ml_converged is False:
ml_opt.run(fmax=fmax*0.90, steps=1)
pos_ml = np.array(guess.positions).flatten()
self.list_interesting_points.append(pos_ml)
pos_ml = apply_mask([pos_ml],
mask_index=self.index_mask)[1]
pred_ml = self.gp.predict(test_fp=pos_ml, uncertainty=True)
energy_ml = pred_ml['prediction'][0][0]
unc_ml = pred_ml['uncertainty_with_reg'][0]
self.list_interesting_energies.append(energy_ml)
self.list_interesting_uncertainties.append(unc_ml)
n_steps_performed += 1
if n_steps_performed > 1000:
if full_output is True:
print('Not converged yet...')
ml_converged = True
if unc_ml >= max_step:
if full_output is True:
print('Maximum uncertainty reach. Early stop.')
ml_converged = True
if ml_opt.converged():
if full_output is True:
print('ML optimized.')
ml_converged = True
# Acquisition functions:
acq_pred = np.array(self.list_interesting_energies)
acq_unc = np.array(self.list_interesting_uncertainties)
if self.acq == 'ucb':
acq_values = UCB(predictions=acq_pred, uncertainty=acq_unc,
objective='min', kappa=-1.0)
if self.acq == 'lcb':
e_minus_unc = np.array(self.list_interesting_energies) - \
np.array(self.list_interesting_uncertainties)
acq_values = -e_minus_unc
if self.acq == 'min_energy':
acq_values = -np.array(self.list_interesting_energies)
max_score = np.argmax(acq_values)
self.interesting_point = self.list_interesting_points[max_score]
# 3. Evaluate and append interesting point.
if full_output is True:
print('Performing evaluation in the real landscape...')
eval_atom = self.ase_ini
pos_atom = self.interesting_point
eval_atom.positions = np.array(pos_atom).reshape((-1, 3))
eval_atom.set_calculator(self.ase_calc)
energy_atom = eval_atom.get_potential_energy()
forces_atom = -eval_atom.get_forces().reshape(-1)
# 4. Convergence and output.
self.list_train.append(pos_atom)
self.list_targets.append(energy_atom)
self.list_gradients.append(forces_atom)
self.list_fmax = get_fmax(np.array([self.list_gradients[-1]]))
self.max_abs_forces = np.max(np.abs(self.list_fmax))
self.list_max_abs_forces.append(self.max_abs_forces)
self.iter += 1
print_info(self)
# Save evaluated image.
self.list_atoms += [eval_atom.copy()]
write(self.filename, self.list_atoms)
# Maximum number of iterations reached.
if self.iter >= steps:
print('Not converged. Maximum number of iterations reached.')
break