def test_gp_gaussian_kernel(self): """Test Gaussian process predictions with the gaussian kernel.""" train_features, train_targets, test_features, test_targets = get_data() # Test prediction routine with gaussian kernel. kdict = {'k1': {'type': 'gaussian', 'width': 1., 'scaling': 1.}} gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_dict=kdict, regularization=1e-3, optimize_hyperparameters=True, scale_data=True) pred = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=True, get_training_error=True, uncertainty=True, epsilon=0.1) mult_dim = pred['prediction'] self.assertEqual(len(pred['prediction']), len(test_features)) print('gaussian prediction (rmse):', pred['validation_error']['rmse_average']) print('gaussian prediction (ins):', pred['validation_error']['insensitive_average']) print('gaussian prediction (abs):', pred['validation_error']['absolute_average']) # Test prediction routine with single width parameter. kdict = { 'k1': { 'type': 'gaussian', 'width': 1., 'scaling': 1., 'dimension': 'single', 'bounds': ((1e-5, None), ), 'scaling_bounds': ((0., None), ) } } gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_dict=kdict, regularization=1e-3, optimize_hyperparameters=True, scale_data=True) self.assertEqual(len(gp.kernel_dict['k1']['width']), 1) pred = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=True, get_training_error=True, uncertainty=True, epsilon=0.1) self.assertEqual(len(pred['prediction']), len(test_features)) self.assertNotEqual(np.sum(pred['prediction']), np.sum(mult_dim)) print('gaussian single width (rmse):', pred['validation_error']['rmse_average'])
def test_gp_addative_kernel(self): """Test Gaussian process predictions with the addative kernel.""" train_features, train_targets, test_features, test_targets = get_data() # Test prediction with addative linear and gaussian kernel. kdict = { 'k1': { 'type': 'linear', 'features': [0, 1], 'scaling': 1. }, 'k2': { 'type': 'gaussian', 'features': [2, 3], 'width': 1., 'scaling': 1. }, 'c1': { 'type': 'constant', 'const': 1. } } gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_dict=kdict, regularization=1e-3, optimize_hyperparameters=True, scale_data=True) pred = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=True, get_training_error=True) self.assertEqual(len(pred['prediction']), len(test_features)) print('addition prediction:', pred['validation_error']['rmse_average'])
def test_gp_laplacian_kernel(self): """Test Gaussian process predictions with the laplacian kernel.""" train_features, train_targets, test_features, test_targets = get_data() # Test prediction routine with laplacian kernel. kdict = [{ 'type': 'laplacian', 'width': 1., 'scaling': 1., 'bounds': ((1e-5, None), ) * np.shape(train_features)[1], 'scaling_bounds': ((0., None), ) }] gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_list=kdict, regularization=np.sqrt(1e-3), optimize_hyperparameters=True, scale_data=True) pred = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=True, get_training_error=True) self.assertEqual(len(pred['prediction']), len(test_features)) print('laplacian prediction:', pred['validation_error']['rmse_average'])
def test_gp_multiplication_kernel(self): """Test Gaussian process predictions with the multiplication kernel.""" train_features, train_targets, test_features, test_targets = get_data() # Test prediction with multiplication of linear & gaussian kernel. kdict = [{ 'type': 'linear', 'features': [0, 1], 'scaling': 1. }, { 'type': 'gaussian', 'features': [2, 3], 'width': 1., 'scaling': 1., 'operation': 'multiplication' }, { 'type': 'constant', 'const': 1. }] gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_list=kdict, regularization=np.sqrt(1e-3), optimize_hyperparameters=True, scale_data=True) pred = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=True, get_training_error=True) self.assertEqual(len(pred['prediction']), len(test_features)) print('multiplication prediction:', pred['validation_error']['rmse_average'])
def test_gp(self): """Test Gaussian process predictions with rescaling.""" train_features, train_targets, test_features, test_targets = get_data() # Test prediction routine with gaussian kernel. kdict = {'k1': {'type': 'gaussian', 'width': 1., 'scaling': 1.}} gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_dict=kdict, regularization=1e-3, optimize_hyperparameters=True, scale_data=True) pred = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=True, get_training_error=True, uncertainty=True) opt = gp.kernel_dict['k1']['width'] orig = hs.rescale_hyperparameters(gp.scaling, gp.kernel_dict)['k1']['width'] assert not np.allclose(opt, orig) scaled = hs.hyperparameters(gp.scaling, gp.kernel_dict)['k1']['width'] assert np.allclose(opt, scaled) print('gaussian prediction (rmse):', pred['validation_error']['rmse_average'])
def test_gp_quadratic_kernel(self): """Test Gaussian process predictions with the quadratic kernel.""" train_features, train_targets, test_features, test_targets = get_data() # Test prediction routine with quadratic kernel. kdict = { 'k1': { 'type': 'quadratic', 'slope': 1., 'degree': 1., 'scaling': 1., 'bounds': ((1e-5, None), ) * (np.shape(train_features)[1] + 1), 'scaling_bounds': ((0., None), ) } } gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_dict=kdict, regularization=1e-3, optimize_hyperparameters=True, scale_data=True) pred = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=True, get_training_error=True) self.assertEqual(len(pred['prediction']), len(test_features)) print('quadratic prediction:', pred['validation_error']['rmse_average'])
def test_gp_linear_kernel(self): """Test Gaussian process predictions with the linear kernel.""" train_features, train_targets, test_features, test_targets = get_data() # Test prediction routine with linear kernel. kdict = { 'k1': { 'type': 'linear', 'scaling': 1., 'scaling_bounds': ((0., None), ) }, 'c1': { 'type': 'constant', 'const': 1. } } gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_dict=kdict, regularization=1e-3, optimize_hyperparameters=True, scale_data=True) pred = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=True, get_training_error=True) self.assertEqual(len(pred['prediction']), len(test_features)) print('linear prediction:', pred['validation_error']['rmse_average'])
def prediction(self, train_features, train_targets, test_features, test_targets): """Ridge regression predictions.""" # Test ridge regression predictions. sigma = 1. kdict = [{'type': 'gaussian', 'width': sigma, 'dimension': 'single'}] regularization = np.sqrt(0.001) gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_list=kdict, regularization=regularization, optimize_hyperparameters=False, scale_data=True) output = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=True, get_training_error=True, uncertainty=True) r = [ np.shape(train_features)[0], output['validation_error']['rmse_average'], output['validation_error']['absolute_average'], np.mean(output['uncertainty']) ] return r
def test_gp_update(self): """Test Gaussian process predictions with the multiplication kernel.""" train_features, train_targets, test_features, test_targets = get_data() kdict = [{ 'type': 'linear', 'scaling': 1. }, { 'type': 'gaussian', 'width': 1., 'scaling': 1., 'operation': 'multiplication' }, { 'type': 'constant', 'const': 1. }] gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_list=kdict, regularization=np.sqrt(1e-3), optimize_hyperparameters=True, scale_data=True) # Test updating the last model. d, f = np.shape(train_features) train_features = np.concatenate((train_features, test_features)) new_features = np.random.random_sample( (np.shape(train_features)[0], 5)) train_features = np.concatenate((train_features, new_features), axis=1) self.assertNotEqual(np.shape(train_features), (d, f)) train_targets = np.concatenate((train_targets, test_targets)) new_features = np.random.random_sample((len(test_features), 5)) test_features = np.concatenate((test_features, new_features), axis=1) kdict = [{ 'type': 'linear', 'scaling': 1. }, { 'type': 'gaussian', 'width': 1., 'scaling': 1., 'operation': 'multiplication' }, { 'type': 'constant', 'const': 1. }] gp.update_gp(train_fp=train_features, train_target=train_targets, kernel_list=kdict) pred = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=True, get_training_error=True) sigma = gp.predict_uncertainty(test_fp=test_features) self.assertEqual(len(sigma['uncertainty']), len(test_features)) self.assertEqual(len(pred['prediction']), len(test_features)) print('Update prediction:', pred['validation_error']['rmse_average'])
def test_acquisition(self): """Test acquisition functions.""" train_features, train_targets, train_atoms, test_features, \ test_targets, test_atoms = self.get_data() # Test prediction routine with gaussian kernel. kdict = {'k1': {'type': 'gaussian', 'width': 1., 'scaling': 1.}} gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_dict=kdict, regularization=1e-3, optimize_hyperparameters=True, scale_data=True) pred = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=True, get_training_error=True, uncertainty=True) print('gaussian prediction (rmse):', pred['validation_error']['rmse_average']) af = AcquisitionFunctions() acq = af.rank( targets=train_targets, predictions=pred['prediction'], uncertainty=pred['uncertainty'], train_features=train_features, test_features=test_features, metrics=['cdf', 'optimistic', 'gaussian', 'UCB', 'EI', 'PI']) self.assertTrue(len(acq['cdf']) == len(pred['prediction'])) self.assertTrue(len(acq['optimistic']) == len(pred['prediction'])) self.assertTrue(len(acq['gaussian']) == len(pred['prediction'])) self.assertTrue(len(acq['UCB']) == len(pred['prediction'])) self.assertTrue(len(acq['EI']) == len(pred['prediction'])) self.assertTrue(len(acq['PI']) == len(pred['prediction'])) acq = af.classify( classifier, train_atoms, test_atoms, targets=train_targets, predictions=pred['prediction'], uncertainty=pred['uncertainty'], train_features=train_features, test_features=test_features, metrics=['cdf', 'optimistic', 'gaussian', 'UCB', 'EI', 'PI']) self.assertTrue(len(acq['cdf']) == len(pred['prediction'])) self.assertTrue(len(acq['optimistic']) == len(pred['prediction'])) self.assertTrue(len(acq['gaussian']) == len(pred['prediction'])) self.assertTrue(len(acq['UCB']) == len(pred['prediction'])) self.assertTrue(len(acq['EI']) == len(pred['prediction'])) self.assertTrue(len(acq['PI']) == len(pred['prediction']))
def minimize_error_time(train_features, train_targets, test_features, test_targets): """A generic fitness function. This fitness function will minimize the cost function as well as the time to train the model. This will provide a Pareto optimial set of solutions upon convergence. Parameters ---------- train_features : array The training features. train_targets : array The training targets. test_features : array The test feaatures. test_targets : array The test targets. """ kernel = { 'k1': { 'type': 'gaussian', 'width': 1., 'scaling': 1., 'dimension': 'single' } } stime = time.time() gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_dict=kernel, regularization=1e-2, optimize_hyperparameters=True, scale_data=True) timing = time.time() - stime pred = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=True, get_training_error=True) score = pred['validation_error']['rmse_average'] return [-score, -timing]
def gp_predict(train_features, train_targets, test_features, test_targets): """Function to perform the GP predictions.""" data = {} kdict = [ {'type': 'gaussian', 'width': 1., 'scaling': 1., 'dimension': 'single'}, ] gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_list=kdict, regularization=1e-2, optimize_hyperparameters=True, scale_data=True) pred = gp.predict(test_fp=test_features) data['result'] = get_error(pred['prediction'], test_targets)['rmse_average'] data['size'] = len(train_targets) return data
def minimize_error_descriptors(train_features, train_targets, test_features, test_targets): """A generic fitness function. This fitness function will minimize the cost function as well as the number of descriptors. This will provide a Pareto optimial set of solutions upon convergence. Parameters ---------- train_features : array The training features. train_targets : array The training targets. test_features : array The test feaatures. test_targets : array The test targets. """ kernel = [{ 'type': 'gaussian', 'width': 1., 'scaling': 1., 'dimension': 'single' }] gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_list=kernel, regularization=1e-2, optimize_hyperparameters=True, scale_data=True) pred = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=True, get_training_error=True) score = pred['validation_error']['rmse_average'] dimension = train_features.shape[1] return [-score, -dimension]
def minimize_error(train_features, train_targets, test_features, test_targets): """A generic fitness function. This fitness function will minimize the cost function. Parameters ---------- train_features : array The training features. train_targets : array The training targets. test_features : array The test feaatures. test_targets : array The test targets. """ kernel = { 'k1': { 'type': 'gaussian', 'width': 1., 'scaling': 1., 'dimension': 'single' } } gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_dict=kernel, regularization=1e-2, optimize_hyperparameters=True, scale_data=True) pred = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=True, get_training_error=True) score = pred['validation_error']['rmse_average'] return [-score]
def fitf(train_features, train_targets, test_features, test_targets): """Define the fitness function for the GA.""" kdict = [{ 'type': 'gaussian', 'width': 1., 'scaling': 1., 'dimension': 'single' }] gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_list=kdict, regularization=1e-2, optimize_hyperparameters=True, scale_data=True) pred = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=True) score = pred['validation_error']['rmse_average'] return -score
def _surrogate_model(train_features, train_targets, test_features=None, test_targets=None): kdict = [{'type': 'gaussian', 'width': 0.5}] gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_list=kdict, regularization=1e-3, optimize_hyperparameters=False, scale_data=True) if test_targets is None: get_validation_error = False else: get_validation_error = True score = gp.predict(test_fp=test_features, test_target=test_targets, get_validation_error=get_validation_error, get_training_error=False, uncertainty=True) to_acquire = np.argsort( probability_density(0., score['prediction'], score['uncertainty'])) return to_acquire, score
# # Example 1: Biased model # sdt1 = 0.001 w1 = 3.0 # too large width results in a biased model kdict = [{"type": "gaussian", "width": w1}] gp = GaussianProcess(kernel_list=kdict, regularization=sdt1, train_fp=std["train"], train_target=train_targets["target"], optimize_hyperparameters=False) under_fit = gp.predict(test_fp=std["test"], uncertainty=True) # Scale predictions back to the original scale under_prediction = np.vstack( under_fit["prediction"]) * train_targets["std"] + train_targets["mean"] under_certainty = np.vstack( under_fit["uncertainty_with_reg"]) * train_targets["std"] # Get average errors error = get_error(under_prediction.reshape(-1), my_func(test).reshape(-1)) # Get confidence interval on predictions upper = under_prediction + under_certainty * tstd lower = under_prediction - under_certainty * tstd plt.figure(0)
# In[5]: fig = plt.figure(figsize=(20, 10)) for w, p in zip([1.5, 1., 0.5, 0.1], [141, 142, 143, 144]): kdict = [{'type': 'gaussian', 'width': w, 'scaling': 1.}] # Set up the prediction routine. gp = GaussianProcess(kernel_list=kdict, regularization=np.sqrt(1e-3), train_fp=train, train_target=target, optimize_hyperparameters=False, scale_data=True) # Do predictions. fit = gp.predict(test_fp=test) # Get average errors. error = get_error(fit['prediction'], afunc(test)) print('Gaussian regression error with {0} width: {1:.3f}'.format( w, error['absolute_average'])) # Plotting. plot(p, fit['prediction']) # ## Scaling Parameter <a name="scaling-parameter"></a> # [(Back to top)](#head) # # And then the same study can be performed with the scaling parameter ($\sigma$). # In[6]:
# In[4]: # Define prediction parameters kdict = [{'type': 'linear', 'scaling': 1.}, {'type': 'constant', 'const': 0.}] # Starting guess for the noise parameter sdt1 = noise_magnitude # Set up the gaussian process. gp1 = GaussianProcess(kernel_list=kdict, regularization=np.sqrt(sdt1), train_fp=std['train'], train_target=train_targets['target'], optimize_hyperparameters=True) # Do predictions. linear = gp1.predict(test_fp=std['test'], uncertainty=True) # Put predictions back on real scale. prediction = np.vstack(linear['prediction']) * train_targets['std'] + train_targets['mean'] # Put uncertainties back on real scale. uncertainty = np.vstack(linear['uncertainty_with_reg']) * train_targets['std'] # Get confidence interval on predictions. over_upper = prediction + uncertainty * tstd over_lower = prediction - uncertainty * tstd # Plotting. plt3d = plt.figure(0).gca(projection='3d') # Plot training data. plt3d.scatter(train[:, 0], train[:, 1], target, color='b') # Plot exact function. plt3d.plot_surface(test_x1, test_x2,
'scaling': 1., 'dimension': 'single' }, { 'type': 'linear', 'scaling': 1. }, ] gp = GaussianProcess(train_fp=train_features, train_target=train_targets, kernel_list=kdict, regularization=1e-2, optimize_hyperparameters=True, scale_data=True) pred = gp.predict(test_fp=test_features) error = get_error(pred['prediction'], test_targets)['rmse_average'] print(error) plt.figure(6) plt.figure(figsize=(30, 15)) plt.plot(test_targets, pred['prediction'], 'o', c='r', alpha=0.5) plt.savefig('gaussian_process.png') # Here we see that the Gaussian process performs slightly worse than the simple ridge regression model. This is to be expected when we are trying to model linear data with a non-linear model. However, the inclusion of the linear kernel results in a good prediction error. If the squared exponential kernel were to be removed from the above example, the resulting model would be the same as the ridge regression model, just trained with the Gaussian process. # # ## Cross-validation <a name="cross-validation"></a> # [(Back to top)](#head) #
}, 'c1': { 'type': 'constant', 'const': 0. } } # Starting guess for the noise parameter sdt1 = noise_magnitude # Set up the gaussian process. gp1 = GaussianProcess(kernel_dict=kdict, regularization=sdt1, train_fp=std['train'], train_target=train_targets['target'], optimize_hyperparameters=True) # Do predictions. linear = gp1.predict(test_fp=std['test'], uncertainty=True) # Put predictions back on real scale. prediction = np.vstack(linear['prediction']) * train_targets['std'] + \ train_targets['mean'] # Put uncertainties back on real scale. uncertainty = np.vstack(linear['uncertainty']) * train_targets['std'] # Get confidence interval on predictions. over_upper = prediction + uncertainty * tstd over_lower = prediction - uncertainty * tstd # Plot the uncertainties upper and lower bounds. plt3d.plot_surface(test_x1, test_x2, over_upper.reshape(np.shape(test_x1)), alpha=0.3, color='r') plt3d.plot_surface(test_x1,
# The scaling parameter $\sigma_v$ is varied in the following. We choose a range of values in the range of `[1., 1e-2, 1e-4, 1e-6]`, iterate over them and plot the predictions from the resulting model. It is seen that as the scaling parameter is decreased, the slope on the predictions also decreases. For the smallest scaling parameter of `1e-6`, there is essentially zero slope and the Gaussian process simply predicts the mean of the data. # In[5]: fig = plt.figure(figsize=(20, 10)) for w, p in zip([1., 1e-2, 1e-4, 1e-6], [141, 142, 143, 144]): kdict = [{'type': 'linear', 'scaling': w}] # Set up the prediction routine. gp = GaussianProcess(kernel_list=kdict, regularization=np.sqrt(1e-3), train_fp=train, train_target=target, optimize_hyperparameters=False, scale_data=True) # Do predictions. fit = gp.predict(test_fp=test, uncertainty=True) # Get average errors. error = get_error(fit['prediction'], afunc(test)) print('Gaussian regression error with {0} width: {1:.3f}'.format( w, error['absolute_average'])) # Plotting. plot(p, fit['prediction']) # ## Regularization parameter <a name="constant-parameter"></a> # [(Back to top)](#head) # # The regularization parameter is varied within the range of `[1., 1e-1, 1e-2, 1e-3]`. Here we find that for larger values the model will under-fit. This will essentially result in the mean of the data being returned across the range of test values. When the regularization parameter gets small enough, it will have little impact on the model predictions as it will be smaller than the noise on the data.
kdict = [{ 'type': 'gaussian', 'width': 1., 'scaling': 1., 'dimension': 'single' }] gp = GaussianProcess(train_fp=train_data, train_target=trainval, kernel_list=kdict, regularization=1e-2, optimize_hyperparameters=True, scale_data=True) pred = gp.predict(test_fp=test_data, test_target=testval, get_validation_error=True, get_training_error=True) score = pred['validation_error']['rmse_average'] print('all features: {0:.3f}'.format(score)) # ## Optimization # # To optimize the feature set, a prediction function is first defined. This will take a boolean array from the genetic algorithm, transform the feature set and test against the holdout data. The error is then calculated and returned for that set of features. The genetic algorithm will aim to maximize the "fitness" of a population of candidates; therefore, the negative of the average cost is returned in this example. # In[6]: def fitf(train_features, train_targets, test_features, test_targets): """Define the fitness function for the GA."""
class GeneralGaussianProcess(object): """Define a general setup for the Gaussin process. This should not be used to try and obtain highly accurate solutions. Though it should give a reasonable model. """ def __init__(self, clean_type='eliminate', dimension='single', kernel='general'): """Initialize the class. Parameters ---------- clean_type : str Define method for handling missing data. Currently only elimination implemented. dimension : str The number of parameters to return. Can be 'single', or 'all'. kernel : str Pass 'smooth' if a smooth but non-linear function is expected. """ self.clean_type = clean_type self.dimension = dimension self.kernel = kernel def train_gaussian_process(self, train_features, train_targets): """Generate a general gaussian process model. Parameters ---------- train_features : array The array of training features. train_targets : array A list of training target values. Returns ------- gp : object The trained Gaussian process. """ train_features, train_targets = self._process_train_data( train_features, train_targets) if 'smooth' in self.kernel: kdict = smooth_kernel(train_features, self.dimension) else: kdict = general_kernel(train_features, self.dimension) self.gp = GaussianProcess( train_fp=train_features, train_target=train_targets, kernel_list=kdict, regularization=1e-2, optimize_hyperparameters=True, scale_data=False ) return self.gp def gaussian_process_predict(self, test_features): """Function to make GP predictions on tests data. Parameters ---------- test_features : array The array of test features. Returns ------- prediction : dict The prediction data generated by the Gaussian process. """ test_features = self.cleaner.transform(test_features) pred = self.gp.predict(test_fp=test_features) return pred def _process_train_data(self, train_features, train_targets): """Prepare the data. Parameters ---------- train_features : array The array of training features. train_targets : array A list of training target values. Returns ------- train_features : array The array of cleaned and scaled training features. train_targets : array A list of cleaned training target values. """ self.cleaner = GeneralPrepreprocess(clean_type=self.clean_type) train_features, train_targets, _ = self.cleaner.process( train_features, train_targets) return train_features, train_targets
'type': 'constant', 'const': 0.0 }] # Starting guess for the noise parameter sdt1 = noise_magnitude # Set up the gaussian process. gp1 = GaussianProcess(kernel_list=kdict, regularization=sdt1, train_fp=std['train'], train_target=train_targets['target'], optimize_hyperparameters=True, scale_optimizer=False) # Do predictions. linear = gp1.predict(test_fp=std['test'], get_validation_error=True, test_target=afunc(test)) prediction = np.array( linear['prediction']) * train_targets['std'] + train_targets['mean'] error = get_error(prediction, afunc(test)) # Plotting. plt3d = plt.figure(1).gca(projection='3d') # Plot training data. plt3d.scatter(train[:, 0], train[:, 1], target, color='b') # Plot exact function. plt3d.plot_surface(test_x1, test_x2, afunc(test).reshape(np.shape(test_x1)), alpha=0.3, color='b') # Plot the prediction.
if True: # Model example 1 - biased model. # Define prediction parameters. sdt1 = 0.001 # Too large width results in a biased model. w1 = 3.0 kdict = {'k1': {'type': 'gaussian', 'width': w1}} # Set up the prediction routine. gp = GaussianProcess(kernel_dict=kdict, regularization=sdt1**2, train_fp=std['train'], train_target=train_targets['target'], optimize_hyperparameters=False) # Do predictions. under_fit = gp.predict(test_fp=std['test'], uncertainty=True) # Scale predictions back to the original scale. under_prediction = np.vstack(under_fit['prediction']) * \ train_targets['std'] + train_targets['mean'] under_uncertainty = np.vstack(under_fit['uncertainty']) * \ train_targets['std'] # Get average errors. error = get_error(under_prediction.reshape(-1), afunc(test).reshape(-1)) print('Gaussian linear regression prediction:', error['absolute_average']) # Get confidence interval on predictions. upper = under_prediction + under_uncertainty * tstd lower = under_prediction - under_uncertainty * tstd # Plot example 1 ax = fig.add_subplot(grid + 1) ax.plot(linex, liney, '-', lw=1, color='black')
class MLMin(object): def __init__(self, x0, prev_calculations=None, restart=False, trajectory='catlearn_opt.traj'): """Optimization setup. Parameters ---------- x0 : Atoms object or trajectory file in ASE format. Initial guess. restart: boolean Restart calculation. Only if a trajectory file with the same name as the one introduced in the 'trajectory' flag is found. This file must be in the same directory than the direction in which the optimization was initiated. prev_calculations: list of Atoms or trajectory file (ASE format). The user can feed previously calculated data for the same hypersurface (i.e. the calculations must be done using the same calculator and parameters). trajectory: string Filename to store the output. """ # General variables. self.prev_calculations = prev_calculations self.filename = trajectory self.iter = 0 self.fmax = 0. self.gp = None self.opt_type = 'MLMin' self.version = self.opt_type + ' ' + __version__ print_version(self.version) if restart is True and prev_calculations is None: if os.path.isfile(self.filename): self.prev_calculations = self.filename msg = 'You must set an initial Atoms structure.' assert x0 is not None, msg # Read trajectory file. if isinstance(x0, str): x0 = read(x0, '-1') self.ase_ini = x0 self.ase_calc = self.ase_ini.get_calculator() msg = 'ASE calculator not found.' assert self.ase_calc, msg self.constraints = self.ase_ini.constraints self.x0 = self.ase_ini.get_positions().flatten() self.num_atoms = self.ase_ini.get_number_of_atoms() # Information from the initial structure. new_atoms = self.ase_ini self.list_atoms = [new_atoms] if self.prev_calculations is not None: if isinstance(self.prev_calculations, str): print('Reading previous calculations from a traj. file.') self.prev_calculations = read(self.prev_calculations, ':') # Check for duplicates. for prev_atoms in self.prev_calculations: duplicate = False prev_pos = prev_atoms.get_positions().reshape(-1) for atoms in self.list_atoms: pos_atoms = atoms.get_positions().reshape(-1) d_ij = euclidean(pos_atoms, prev_pos) if d_ij == 0: duplicate = True if duplicate is False: self.list_atoms += [prev_atoms] # Extract information from previous calculations. self.list_train = [] self.list_targets = [] self.list_gradients = [] for i in range(0, len(self.list_atoms)): pos_atoms = list(self.list_atoms[i].get_positions().reshape(-1)) energy_atoms = self.list_atoms[i].get_potential_energy() grad_atoms = list(-self.list_atoms[i].get_forces().reshape(-1)) self.list_train.append(pos_atoms) self.list_targets.append(energy_atoms) self.list_gradients.append(grad_atoms) self.list_max_abs_forces = [] for i in self.list_gradients: self.list_fmax = get_fmax(np.array([i])) self.max_abs_forces = np.max(np.abs(self.list_fmax)) self.list_max_abs_forces.append(self.max_abs_forces) # Get constraints. self.index_mask = create_mask(self.ase_ini, self.constraints) # Dump all Atoms to file. write(self.filename, self.list_atoms) print_info(self) print(self.list_targets) def run(self, fmax=0.05, steps=200, kernel='SQE', max_step=0.25, acq='min_energy', full_output=False): """Executing run will start the optimization process. Parameters ---------- fmax : float Convergence criteria (in eV/Angstrom). steps : int Max. number of optimization steps. kernel: string Type of covariance function to be used. Implemented are: SQE (fixed hyperparamters), SQE_opt and ARD_SQE. max_step: float Early stopping criteria. Maximum uncertainty before stopping the optimization in the predicted landscape. acq : string The acquisition function that decides the next point to evaluate. Implemented are: 'lcb', 'ucb', 'min_energy'. full_output: boolean Whether to print on screen the full output (True) or not (False). Returns ------- Optimized atom structure. """ self.fmax = fmax self.acq = acq success_hyper = False while not converged(self): # 1. Train Machine Learning model. train = np.copy(self.list_train) targets = np.copy(self.list_targets) gradients = np.copy(self.list_gradients) self.u_prior = np.max(targets) scaled_targets = targets - self.u_prior sigma_f = 1e-3 + np.std(scaled_targets)**2 if kernel == 'SQE_fixed': opt_hyper = False kdict = [{'type': 'gaussian', 'width': 0.4, 'dimension': 'single', 'bounds': ((0.4, 0.4),), 'scaling': 1.0, 'scaling_bounds': ((1.0, 1.0),)}, {'type': 'noise_multi', 'hyperparameters': [0.005 * 0.4**2, 0.005], 'bounds': ((0.005 * 0.4**2, 0.005 * 0.4**2), (0.005, 0.005),)} ] if kernel == 'SQE': opt_hyper = True kdict = [{'type': 'gaussian', 'width': 0.4, 'dimension': 'single', 'bounds': ((0.01, 1.0),), 'scaling': sigma_f, 'scaling_bounds': ((sigma_f, sigma_f),)}, {'type': 'noise_multi', 'hyperparameters': [0.005, 0.005 * 0.4**2], 'bounds': ((0.001, 0.050), (0.001 * 0.4**2, 0.050),)} ] if kernel == 'ARD_SQE': opt_hyper = True kdict = [{'type': 'gaussian', 'width': 0.4, 'dimension': 'features', 'bounds': ((0.01, 1.0),) * len(self.index_mask), 'scaling': sigma_f, 'scaling_bounds': ((sigma_f, sigma_f),)}, {'type': 'noise_multi', 'hyperparameters': [0.005, 0.0005], 'bounds': ((0.001, 0.005), (0.0005, 0.002),)} ] if success_hyper is not False: kdict = success_hyper if self.index_mask is not None: train = apply_mask(list_to_mask=train, mask_index=self.index_mask)[1] gradients = apply_mask(list_to_mask=gradients, mask_index=self.index_mask)[1] print('Training a GP process...') print('Number of training points:', len(scaled_targets)) train = train.tolist() gradients = gradients.tolist() self.gp = GaussianProcess(kernel_list=kdict, regularization=0.0, regularization_bounds=(0.0, 0.0), train_fp=train, train_target=scaled_targets, gradients=gradients, optimize_hyperparameters=False) print('GP process trained.') if opt_hyper is True: if len(self.list_targets) > 5: self.gp.optimize_hyperparameters() if self.gp.theta_opt.success is True: if full_output is True: print('Hyperparam. optimization was successful.') print('Updating kernel list...') success_hyper = self.gp.kernel_list if self.gp.theta_opt.success is not True: if full_output is True: print('Hyperparam. optimization unsuccessful.') if success_hyper is False: if full_output is True: print('Not enough data...') if success_hyper is not False: if full_output is True: print('Using the last optimized ' 'hyperparamters.') if full_output is True: print('Kernel list:', self.gp.kernel_list) # 2. Optimize Machine Learning model. self.list_interesting_points = [] self.list_interesting_energies = [] self.list_interesting_uncertainties = [] guess = self.ase_ini guess_pos = np.array(self.list_train[-1]) guess.positions = guess_pos.reshape(-1, 3) guess.set_calculator(ASECalc(gp=self.gp, index_constraints=self.index_mask, scaling_targets=self.u_prior) ) # Optimization in the predicted landscape: ml_opt = MDMin(guess, trajectory=None, logfile=None, dt=0.020) if full_output is True: print('Starting optimization on the predicted landscape...') ml_converged = False n_steps_performed = 0 while ml_converged is False: ml_opt.run(fmax=fmax*0.90, steps=1) pos_ml = np.array(guess.positions).flatten() self.list_interesting_points.append(pos_ml) pos_ml = apply_mask([pos_ml], mask_index=self.index_mask)[1] pred_ml = self.gp.predict(test_fp=pos_ml, uncertainty=True) energy_ml = pred_ml['prediction'][0][0] unc_ml = pred_ml['uncertainty_with_reg'][0] self.list_interesting_energies.append(energy_ml) self.list_interesting_uncertainties.append(unc_ml) n_steps_performed += 1 if n_steps_performed > 1000: if full_output is True: print('Not converged yet...') ml_converged = True if unc_ml >= max_step: if full_output is True: print('Maximum uncertainty reach. Early stop.') ml_converged = True if ml_opt.converged(): if full_output is True: print('ML optimized.') ml_converged = True # Acquisition functions: acq_pred = np.array(self.list_interesting_energies) acq_unc = np.array(self.list_interesting_uncertainties) if self.acq == 'ucb': acq_values = UCB(predictions=acq_pred, uncertainty=acq_unc, objective='min', kappa=-1.0) if self.acq == 'lcb': e_minus_unc = np.array(self.list_interesting_energies) - \ np.array(self.list_interesting_uncertainties) acq_values = -e_minus_unc if self.acq == 'min_energy': acq_values = -np.array(self.list_interesting_energies) max_score = np.argmax(acq_values) self.interesting_point = self.list_interesting_points[max_score] # 3. Evaluate and append interesting point. if full_output is True: print('Performing evaluation in the real landscape...') eval_atom = self.ase_ini pos_atom = self.interesting_point eval_atom.positions = np.array(pos_atom).reshape((-1, 3)) eval_atom.set_calculator(self.ase_calc) energy_atom = eval_atom.get_potential_energy() forces_atom = -eval_atom.get_forces().reshape(-1) # 4. Convergence and output. self.list_train.append(pos_atom) self.list_targets.append(energy_atom) self.list_gradients.append(forces_atom) self.list_fmax = get_fmax(np.array([self.list_gradients[-1]])) self.max_abs_forces = np.max(np.abs(self.list_fmax)) self.list_max_abs_forces.append(self.max_abs_forces) self.iter += 1 print_info(self) # Save evaluated image. self.list_atoms += [eval_atom.copy()] write(self.filename, self.list_atoms) # Maximum number of iterations reached. if self.iter >= steps: print('Not converged. Maximum number of iterations reached.') break