def kernel_Matern(k=1e0, l_t=5e3, l_l=5e3):
kernel = C(k, (1e-2, 1e4)) * Matern(
(l_t, l_l), ((1e0, 1e6), (1e0, 1e6)), nu=1.5)
return kernel
def fit(self, X, y):
"""Fit Gaussian process regression model.
Parameters
----------
X : array-like, shape = (n_samples, n_features)
Training data
y : array-like, shape = (n_samples, [n_output_dims])
Target values
Returns
-------
self : returns an instance of self.
"""
if self.kernel is None: # Use an RBF kernel as default
self.kernel_ = C(1.0, constant_value_bounds="fixed") \
* RBF(1.0, length_scale_bounds="fixed")
else:
self.kernel_ = clone(self.kernel)
self._rng = check_random_state(self.random_state)
X, y = check_X_y(X, y, multi_output=True, y_numeric=True)
# Normalize target value
if self.normalize_y:
self._y_train_mean = np.mean(y, axis=0)
# demean y
y = y - self._y_train_mean
else:
self._y_train_mean = np.zeros(1)
if np.iterable(self.alpha) \
and self.alpha.shape[0] != y.shape[0]:
if self.alpha.shape[0] == 1:
self.alpha = self.alpha[0]
else:
raise ValueError(
"alpha must be a scalar or an array"
" with same number of entries as y.(%d != %d)" %
(self.alpha.shape[0], y.shape[0]))
self.X_train_ = np.copy(X) if self.copy_X_train else X
self.y_train_ = np.copy(y) if self.copy_X_train else y
if self.optimizer is not None and self.kernel_.n_dims > 0:
# Choose hyperparameters based on maximizing the log-marginal
# likelihood (potentially starting from several initial values)
def obj_func(theta, eval_gradient=True):
if eval_gradient:
lml, grad = self.log_marginal_likelihood(
theta, eval_gradient=True)
return -lml, -grad
else:
return -self.log_marginal_likelihood(theta)
# First optimize starting from theta specified in kernel
optima = [(self._constrained_optimization(obj_func,
self.kernel_.theta,
self.kernel_.bounds))]
# Additional runs are performed from log-uniform chosen initial
# theta
if self.n_restarts_optimizer > 0:
if not np.isfinite(self.kernel_.bounds).all():
raise ValueError(
"Multiple optimizer restarts (n_restarts_optimizer>0) "
"requires that all bounds are finite.")
bounds = self.kernel_.bounds
for iteration in range(self.n_restarts_optimizer):
theta_initial = \
self._rng.uniform(bounds[:, 0], bounds[:, 1])
optima.append(
self._constrained_optimization(obj_func, theta_initial,
bounds))
# Select result from run with minimal (negative) log-marginal
# likelihood
lml_values = list(map(itemgetter(1), optima))
self.kernel_.theta = optima[np.argmin(lml_values)][0]
self.log_marginal_likelihood_value_ = -np.min(lml_values)
else:
self.log_marginal_likelihood_value_ = \
self.log_marginal_likelihood(self.kernel_.theta)
# Precompute quantities required for predictions which are independent
# of actual query points
K = self.kernel_(self.X_train_)
K[np.diag_indices_from(K)] += self.alpha
try:
self.L_ = cholesky(K, lower=True) # Line 2
# self.L_ changed, self._K_inv needs to be recomputed
self._K_inv = None
except np.linalg.LinAlgError as exc:
exc.args = ("The kernel, %s, is not returning a "
"positive definite matrix. Try gradually "
"increasing the 'alpha' parameter of your "
"GaussianProcessRegressor estimator." %
self.kernel_, ) + exc.args
raise
self.alpha_ = cho_solve((self.L_, True), self.y_train_) # Line 3
return self
return x * np.sin(x) # ---------------------------------------------------------------------- # First the noiseless case X = np.atleast_2d([1., 3., 5., 6., 7., 8.]).T # Observations y = f(X).ravel() # Mesh the input space for evaluations of the real function, the prediction and # its MSE x = np.atleast_2d(np.linspace(0, 10, 1000)).T # Instantiate a Gaussian Process model kernel = C(1.0, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2)) gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9) # Fit to data using Maximum Likelihood Estimation of the parameters gp.fit(X, y) # Make the prediction on the meshed x-axis (ask for MSE as well) y_pred, sigma = gp.predict(x, return_std=True) # Plot the function, the prediction and the 95% confidence interval based on # the MSE plt.figure() plt.plot(x, f(x), 'r:', label=u'$f(x) = x\,\sin(x)$') plt.plot(X, y, 'r.', markersize=10, label=u'Observations') plt.plot(x, y_pred, 'b-', label=u'Prediction') plt.fill(np.concatenate([x, x[::-1]]),
#！/usr/bin/env python
#-*-coding:utf-8-*-
#user:xieyuhui
import numpy as np
from sklearn.gaussian_process.kernels import RBF, Matern, RationalQuadratic, ExpSineSquared, DotProduct, ConstantKernel as C
import pylab
from sklearn.gaussian_process import GaussianProcessRegressor
#X_ = np.array([[0], [1.1], [1.3], [2.2], [2.8], [3.6], [3.7], [4.6], [4.7], [4.8]])
#Y_ = np.array([[-0.1], [0.9], [1], [0.1], [0], [0.7], [0.8], [-1.1], [-1], [-0.8]])
X = np.linspace(0, 5, 100)[:, None]
# RBF
kernel = C(1.0) * RBF(length_scale=1)
gp = GaussianProcessRegressor(kernel=kernel,
alpha=1e-5,
n_restarts_optimizer=10)
pylab.figure(0, figsize=(14, 12))
pylab.subplot(3, 2, 1)
ymean, y_std = gp.predict(X, return_std=True)
pylab.plot(X, ymean, 'k', lw=3, zorder=9, label="mean")
pylab.fill_between(X[:, 0], ymean - y_std, ymean + y_std, alpha=0.5, color='k')
y_samples = gp.sample_y(X, 10)
pylab.plot(X, y_samples, color='b', lw=2)
pylab.plot(X, y_samples[:, 0], color='b', lw=2, label="sample")
pylab.legend(loc="best")
pylab.xlim(0, 5)
pylab.ylim(-3, 3)
pylab.title("Prior Samples")
# reload = True
reload = False
n_iter = 1000
N_EARLY_STOPPING = 1000
# ALPHA = MEAN # prior:
ALPHA = 0.001 # ndim ** 1
GAMMA = 10**(-2) * 2 * ndim
GAMMA0 = 0.01 * GAMMA
GAMMA_Y = 10**(-2) # weight of adjacen
IS_EDGE_NORMALIZED = True
# kernel = Matern(nu=2.5)
kernel = C(1) * RBF(2)
# kernel = None
BURNIN = False # TODO
INITIAL_K = 10
INITIAL_THETA = 10
UPDATE_HYPERPARAM_FUNC = 'pairwise_sampling' # None
output_dir = 'output' # _gmrf_min0max1_easy'
parameter_dir = os.path.join('param_dir', 'csv_files')
result_filename = os.path.join(output_dir, 'gaussian_result_2dim.csv')
ACQUISITION_FUNC = 'ts' # 'ei'
ACQUISITION_PARAM_DIC = {
'beta': 5, #for "ucb"
def parameterized_inference(algorithm='carl',
morphing_aware=False,
training_sample='baseline',
use_smearing=False,
denominator=0,
alpha=None,
training_sample_size=None,
do_neyman=False,
options=''):
"""
Trains and evaluates one of the parameterized higgs_inference methods.
Input:
training_sample_size:
denominator:
algorithm: Type of the algorithm used. Currently supported:
'carl', 'score', 'combined', 'regression', and
'combinedregression'.
morphing_aware: bool that decides whether a morphing-aware or
morphing-agnostic architecture is used.
training_sample: Training sample. Can be 'baseline', 'basis', or 'random'.
use_smearing:
alpha: Factor that weights the score term in the if algorithm
is 'combined' or 'combinedregression'.
do_neyman: Switches on the evaluation of toy experiments for
the Neyman construction.
options: Further options in a list of strings or string.
Output:
"""
logging.info('----> Starting parameterized inference <----')
assert algorithm in ['carl', 'score', 'combined', 'regression', 'combinedregression']
assert training_sample in ['baseline', 'basis', 'random']
################################################################################
# Settings
################################################################################
random_theta_mode = training_sample == 'random'
basis_theta_mode = training_sample == 'basis'
learn_logr_mode = ('learns' not in options)
new_sample_mode = ('new' in options)
short_mode = ('short' in options)
long_mode = ('long' in options)
deep_mode = ('deep' in options)
shallow_mode = ('shallow' in options)
debug_mode = ('debug' in options)
factor_out_sm_in_aware_mode = morphing_aware and ('factorsm' in options)
small_lr_mode = ('slowlearning' in options)
large_lr_mode = ('fastlearning' in options)
large_batch_mode = ('largebatch' in options)
small_batch_mode = ('smallbatch' in options)
constant_lr_mode = ('constantlr' in options)
neyman2_mode = ('neyman2' in options)
neyman3_mode = ('neyman3' in options)
recalibration_mode = ('recalibration' in options)
filename_addition = ''
if morphing_aware:
filename_addition = '_aware'
if random_theta_mode:
filename_addition += '_random'
elif basis_theta_mode:
filename_addition += '_basis'
if not learn_logr_mode:
filename_addition += '_learns'
if factor_out_sm_in_aware_mode:
filename_addition += '_factorsm'
learning_rate = settings.learning_rate_default
if small_lr_mode:
filename_addition += '_slowlearning'
learning_rate = settings.learning_rate_small
elif large_lr_mode:
filename_addition += '_fastlearning'
learning_rate = settings.learning_rate_large
lr_decay = settings.learning_rate_decay
if constant_lr_mode:
lr_decay = 0.
filename_addition += '_constantlr'
batch_size = settings.batch_size_default
if large_batch_mode:
filename_addition += '_largebatch'
batch_size = settings.batch_size_large
elif small_batch_mode:
filename_addition += '_smallbatch'
batch_size = settings.batch_size_small
settings.batch_size = batch_size
alpha_regression = settings.alpha_regression_default
alpha_carl = settings.alpha_carl_default
if alpha is not None:
alpha_regression = alpha
alpha_carl = alpha
precision = int(max(- math.floor(np.log10(alpha)) + 1, 0))
filename_addition += '_alpha_' + format_number(alpha, precision)
n_hidden_layers = settings.n_hidden_layers_default
if shallow_mode:
n_hidden_layers = settings.n_hidden_layers_shallow
filename_addition += '_shallow'
elif deep_mode:
n_hidden_layers = settings.n_hidden_layers_deep
filename_addition += '_deep'
n_epochs = settings.n_epochs_default
early_stopping = True
early_stopping_patience = settings.early_stopping_patience
if debug_mode:
n_epochs = settings.n_epochs_short
early_stopping = False
filename_addition += '_debug'
elif long_mode:
n_epochs = settings.n_epochs_long
filename_addition += '_long'
elif short_mode:
n_epochs = settings.n_epochs_short
early_stopping = False
filename_addition += '_short'
if training_sample_size is not None:
filename_addition += '_trainingsamplesize_' + str(training_sample_size)
n_epoch_factor = int(len(settings.thetas_train) * \
(settings.n_events_baseline_num + \
settings.n_events_baseline_den)/training_sample_size)
n_epochs *= n_epoch_factor
lr_decay /= float(n_epoch_factor)
early_stopping_patience *= n_epoch_factor
input_X_prefix = ''
if use_smearing:
input_X_prefix = 'smeared_'
filename_addition += '_smeared'
theta1 = settings.theta1_default
input_filename_addition = ''
if denominator > 0:
input_filename_addition = '_denom' + str(denominator)
filename_addition += '_denom' + str(denominator)
theta1 = settings.theta1_alternatives[denominator - 1]
if new_sample_mode:
filename_addition += '_new'
input_filename_addition += '_new'
n_expected_events_neyman = settings.n_expected_events_neyman
n_neyman_null_experiments = settings.n_neyman_null_experiments
n_neyman_alternate_experiments = settings.n_neyman_alternate_experiments
neyman_filename = 'neyman'
if neyman2_mode:
neyman_filename = 'neyman2'
n_expected_events_neyman = settings.n_expected_events_neyman2
n_neyman_null_experiments = settings.n_neyman2_null_experiments
n_neyman_alternate_experiments = settings.n_neyman2_alternate_experiments
if neyman3_mode:
neyman_filename = 'neyman3'
n_expected_events_neyman = settings.n_expected_events_neyman3
n_neyman_null_experiments = settings.n_neyman3_null_experiments
n_neyman_alternate_experiments = settings.n_neyman3_alternate_experiments
results_dir = settings.base_dir + '/results/parameterized'
neyman_dir = settings.neyman_dir + '/parameterized'
logging.info('Main settings:')
logging.info(' Algorithm: %s', algorithm)
logging.info(' Morphing-aware: %s', morphing_aware)
logging.info(' Training sample: %s', training_sample)
logging.info(' Denominator theta: denominator %s = theta %s = %s',
denominator, theta1, settings.thetas[theta1])
logging.info('Options:')
logging.info(' Number of hidden layers: %s', n_hidden_layers)
if algorithm == 'combined':
logging.info(' alpha: %s', alpha_carl)
elif algorithm == 'combinedregression':
logging.info(' alpha: %s', alpha_regression)
logging.info(' Batch size: %s', batch_size)
logging.info(' Learning rate: %s', learning_rate)
logging.info(' Learning rate decay: %s', lr_decay)
logging.info(' Number of epochs: %s', n_epochs)
logging.info(' Training samples: %s', 'all' if training_sample_size is None else training_sample_size)
if do_neyman:
logging.info(' NC experiments: (%s alternate + %s null) ' + \
'experiments with %s alternate events each',
n_neyman_alternate_experiments, n_neyman_null_experiments,
n_expected_events_neyman)
else:
logging.info(' NC experiments: False')
logging.info(' Debug mode: %s', debug_mode)
################################################################################
# Data
################################################################################
# Load data
train_filename = '_train'
if random_theta_mode:
train_filename += '_random'
elif basis_theta_mode:
train_filename += '_basis'
train_filename += input_filename_addition
X_train = np.load(settings.unweighted_events_dir + '/' + \
input_X_prefix + 'X' + train_filename + '.npy')
X_train_unshuffled = np.load(settings.unweighted_events_dir + '/' + \
input_X_prefix + 'X' + train_filename + '.npy')
y_train = np.load(settings.unweighted_events_dir + '/y' + train_filename + '.npy')
scores_train = np.load(settings.unweighted_events_dir + '/scores' + \
train_filename + '.npy')
r_train = np.load(settings.unweighted_events_dir + '/r' + train_filename + '.npy')
theta0_train = np.load(settings.unweighted_events_dir + '/theta0' + \
train_filename + '.npy')
theta0_train_unshuffled = np.load(settings.unweighted_events_dir + \
'/theta0' + train_filename + '.npy')
X_calibration = np.load(settings.unweighted_events_dir + '/' + \
input_X_prefix + 'X_calibration' + \
input_filename_addition + '.npy')
weights_calibration = np.load(settings.unweighted_events_dir + \
'/weights_calibration' + \
input_filename_addition + '.npy')
if recalibration_mode:
X_recalibration = np.load(
settings.unweighted_events_dir + '/' + input_X_prefix + 'X_recalibration' + '.npy')
X_test = np.load(
settings.unweighted_events_dir + '/' + input_X_prefix + 'X_test' + input_filename_addition + '.npy')
r_test = np.load(settings.unweighted_events_dir + '/r_test' + input_filename_addition + '.npy')
X_roam = np.load(
settings.unweighted_events_dir + '/' + input_X_prefix + 'X_roam' + input_filename_addition + '.npy')
n_roaming = len(X_roam)
X_illustration = np.load(
settings.unweighted_events_dir + '/' + input_X_prefix + 'X_illustration' + input_filename_addition + '.npy')
if do_neyman:
X_neyman_alternate = np.load(
settings.unweighted_events_dir + '/neyman/' + input_X_prefix + 'X_' + neyman_filename + '_alternate.npy')
n_events_test = X_test.shape[0]
assert settings.n_thetas == r_test.shape[0]
# Shuffle training data
X_train, y_train, scores_train, r_train, theta0_train = shuffle(X_train,
y_train,
scores_train,
r_train,
theta0_train,
random_state=44)
# Limit training sample size
keras_verbosity = 2
if training_sample_size is not None:
keras_verbosity = 0
original_training_sample_size = X_train.shape[0]
X_train = X_train[:training_sample_size]
y_train = y_train[:training_sample_size]
scores_train = scores_train[:training_sample_size]
r_train = r_train[:training_sample_size]
theta0_train = theta0_train[:training_sample_size]
logging.info('Reduced training sample size from %s to %s (factor %s)',
original_training_sample_size, X_train.shape[0], n_epoch_factor)
# Normalize data
scaler = StandardScaler()
scaler.fit(np.array(X_train, dtype=np.float64))
X_train_transformed = scaler.transform(X_train)
X_train_transformed_unshuffled = scaler.transform(X_train_unshuffled)
X_test_transformed = scaler.transform(X_test)
X_roam_transformed = scaler.transform(X_roam)
X_calibration_transformed = scaler.transform(X_calibration)
X_illustration_transformed = scaler.transform(X_illustration)
if recalibration_mode:
X_recalibration_transformed = scaler.transform(X_recalibration)
if do_neyman:
X_neyman_alternate_transformed = scaler.transform(
X_neyman_alternate.reshape((-1, X_neyman_alternate.shape[2])))
# Roaming data
X_thetas_train = np.hstack((X_train_transformed, theta0_train))
X_thetas_train_unshuffled = np.hstack((X_train_transformed_unshuffled, theta0_train_unshuffled))
y_logr_score_train = np.hstack((y_train.reshape(-1, 1), np.log(r_train).reshape((-1, 1)), scores_train))
xi = np.linspace(-1.0, 1.0, settings.n_thetas_roam)
yi = np.linspace(-1.0, 1.0, settings.n_thetas_roam)
xx, yy = np.meshgrid(xi, yi)
thetas_roam = np.asarray((xx.flatten(), yy.flatten())).T
X_thetas_roam = []
for i in range(n_roaming):
X_thetas_roam.append(np.zeros((settings.n_thetas_roam ** 2, X_roam_transformed.shape[1] + 2)))
X_thetas_roam[-1][:, :-2] = X_roam_transformed[i, :]
X_thetas_roam[-1][:, -2:] = thetas_roam
if debug_mode:
X_thetas_train = X_thetas_train[::100]
y_logr_score_train = y_logr_score_train[::100]
X_test_transformed = X_test[::100]
r_test = r_test[:, ::100]
X_calibration_transformed = X_calibration_transformed[::100]
weights_calibration = weights_calibration[:, ::100]
X_illustration_transformed = X_illustration_transformed[::100]
if recalibration_mode:
X_recalibration_transformed = X_recalibration_transformed[::100]
n_events_test = len(X_test_transformed)
################################################################################
# Training
################################################################################
if algorithm == 'carl':
if morphing_aware:
regr = KerasRegressor(lambda:
make_classifier_carl_morphingaware(n_hidden_layers=n_hidden_layers,
learn_log_r=learn_logr_mode,
learning_rate=learning_rate),
epochs=n_epochs, validation_split=settings.validation_split,
verbose=keras_verbosity)
else:
# Agnostic Parametrization ???
regr = KerasRegressor(lambda:
make_classifier_carl(n_hidden_layers=n_hidden_layers,
learn_log_r=learn_logr_mode,
learning_rate=learning_rate),
epochs=n_epochs, validation_split=settings.validation_split,
verbose=keras_verbosity)
elif algorithm == 'score':
if morphing_aware:
regr = KerasRegressor(lambda:
make_classifier_score_morphingaware(n_hidden_layers=n_hidden_layers,
learn_log_r=learn_logr_mode,
learning_rate=learning_rate),
epochs=n_epochs, validation_split=settings.validation_split,
verbose=keras_verbosity)
else:
# Agnostic Parametrization ???
regr = KerasRegressor(lambda:
make_classifier_score(n_hidden_layers=n_hidden_layers,
learn_log_r=learn_logr_mode,
learning_rate=learning_rate),
epochs=n_epochs, validation_split=settings.validation_split,
verbose=keras_verbosity)
elif algorithm == 'combined':
if morphing_aware:
regr = KerasRegressor(lambda:
make_classifier_combined_morphingaware(n_hidden_layers=n_hidden_layers,
learn_log_r=learn_logr_mode,
alpha=alpha_carl,
learning_rate=learning_rate),
epochs=n_epochs, validation_split=settings.validation_split,
verbose=keras_verbosity)
else:
regr = KerasRegressor(lambda:
make_classifier_combined(n_hidden_layers=n_hidden_layers,
learn_log_r=learn_logr_mode,
alpha=alpha_carl,
learning_rate=learning_rate),
epochs=n_epochs, validation_split=settings.validation_split,
verbose=keras_verbosity)
elif algorithm == 'regression':
if morphing_aware:
regr = KerasRegressor(lambda:
make_regressor_morphingaware(n_hidden_layers=n_hidden_layers,
factor_out_sm=factor_out_sm_in_aware_mode,
learning_rate=learning_rate),
epochs=n_epochs, validation_split=settings.validation_split,
verbose=keras_verbosity)
else:
regr = KerasRegressor(lambda:
make_regressor(n_hidden_layers=n_hidden_layers),
epochs=n_epochs, validation_split=settings.validation_split,
verbose=keras_verbosity)
elif algorithm == 'combinedregression':
if morphing_aware:
regr = KerasRegressor(lambda:
make_combined_regressor_morphingaware(n_hidden_layers=n_hidden_layers,
factor_out_sm=factor_out_sm_in_aware_mode,
alpha=alpha_regression,
learning_rate=learning_rate),
epochs=n_epochs, validation_split=settings.validation_split,
verbose=keras_verbosity)
else:
regr = KerasRegressor(lambda:
make_combined_regressor(n_hidden_layers=n_hidden_layers,
alpha=alpha_regression,
learning_rate=learning_rate),
epochs=n_epochs, validation_split=settings.validation_split,
verbose=keras_verbosity)
else:
raise ValueError()
# Callbacks
callbacks = []
detailed_history = {}
callbacks.append(DetailedHistory(detailed_history))
if not constant_lr_mode:
def lr_scheduler(epoch):
return learning_rate * np.exp(- epoch * lr_decay)
callbacks.append(LearningRateScheduler(lr_scheduler))
if early_stopping:
callbacks.append(EarlyStopping(verbose=1, patience=early_stopping_patience))
# Training
logging.info('Starting training')
history = regr.fit(X_thetas_train[::], y_logr_score_train[::],
callbacks=callbacks, batch_size=batch_size)
# Save metrics
def _save_metrics(key, filename):
try:
metrics = np.asarray([history.history[key],
history.history['val_' + key]])
np.save(results_dir + '/traininghistory_' + filename + '_' + \
algorithm + filename_addition + '.npy', metrics)
except KeyError:
logging.warning('Key %s not found in per-epoch history. ' + \
'Available keys: %s',
key, list(history.history.keys()))
try:
detailed_metrics = np.asarray(detailed_history[key])
np.save(results_dir + '/traininghistory_100batches_' + \
filename + '_' + algorithm + filename_addition + \
'.npy', detailed_metrics)
except KeyError:
logging.warning('Key %s not found in per-batch history. ' + \
'Available keys: %s',
key, list(detailed_history.keys()))
_save_metrics('loss', 'loss')
_save_metrics('full_cross_entropy', 'ce')
_save_metrics('full_mse_log_r', 'mse_logr')
_save_metrics('full_mse_score', 'mse_scores')
# Evaluate r_hat on training sample
r_hat_train = np.exp(regr.predict(X_thetas_train_unshuffled)[:, 1])
# np.save(results_dir + '/r_train_' + algorithm + filename_addition + '.npy', r_hat_train)
################################################################################
# Raw evaluation loop
################################################################################
# carl wrapper
# ratio = ClassifierScoreRatio(regr, prefit=True)
logging.info('Starting evaluation')
expected_llr = []
mse_log_r = []
trimmed_mse_log_r = []
eval_times = []
expected_r_vs_sm = []
if recalibration_mode:
recalibration_expected_r = []
for t, theta in enumerate(settings.thetas):
if (t + 1) % 100 == 0:
logging.info('Starting theta %s / %s', t + 1, settings.n_thetas)
################################################################################
# Evaluation
################################################################################
# Prepare test data
thetas0_array = np.zeros((X_test_transformed.shape[0], 2),
dtype=X_test_transformed.dtype)
thetas0_array[:, :] = theta
X_thetas_test = np.hstack((X_test_transformed, thetas0_array))
# Evaluation
time_before = time.time()
prediction = regr.predict(X_thetas_test)
eval_times.append(time.time() - time_before)
this_log_r = prediction[:, 1]
this_score = prediction[:, 2:4]
if morphing_aware:
this_wi = prediction[:, 4:19]
this_ri = prediction[:, 19:]
logging.debug('Morphing weights for theta %s (%s): %s',
t, theta, this_wi[0])
# Extract numbers of interest
expected_llr.append(- 2.*settings.n_expected_events/ \
n_events_test*np.sum(this_log_r))
mse_log_r.append(calculate_mean_squared_error(np.log(r_test[t]),
this_log_r, 0.))
trimmed_mse_log_r.append(calculate_mean_squared_error(np.log(r_test[t]),
this_log_r, 'auto'))
if t == settings.theta_observed:
r_sm = np.exp(this_log_r)
expected_r_vs_sm.append(np.mean(np.exp(this_log_r) / r_sm))
# For benchmark thetas, save more info
if t == settings.theta_benchmark_nottrained:
np.save(results_dir + '/r_nottrained_' + algorithm + \
filename_addition + '.npy', np.exp(this_log_r))
np.save(results_dir + '/scores_nottrained_' + algorithm + \
filename_addition + '.npy', this_score)
np.save(results_dir + '/r_vs_sm_nottrained_' + algorithm + \
filename_addition + '.npy', np.exp(this_log_r) / r_sm)
if morphing_aware:
np.save(results_dir + '/morphing_ri_nottrained_' + \
algorithm + filename_addition + '.npy', this_ri)
np.save(results_dir + '/morphing_wi_nottrained_' + \
algorithm + filename_addition + '.npy', this_wi)
elif t == settings.theta_benchmark_trained:
np.save(results_dir + '/r_trained_' + algorithm + \
filename_addition + '.npy', np.exp(this_log_r))
np.save(results_dir + '/scores_trained_' + algorithm + \
filename_addition + '.npy', this_score)
np.save(results_dir + '/r_vs_sm_trained_' + algorithm + \
filename_addition + '.npy', np.exp(this_log_r) / r_sm)
if morphing_aware:
np.save(results_dir + '/morphing_ri_trained_' + \
algorithm + filename_addition + '.npy', this_ri)
np.save(results_dir + '/morphing_wi_trained_' + \
algorithm + filename_addition + '.npy', this_wi)
################################################################################
# Recalibration
################################################################################
if recalibration_mode:
# Prepare data for recalibration
thetas0_array = np.zeros((X_recalibration_transformed.shape[0], 2),
dtype=X_recalibration_transformed.dtype)
thetas0_array[:, :] = settings.thetas[t]
X_thetas_recalibration = np.hstack((X_recalibration_transformed, thetas0_array))
# Evaluate recalibration data
prediction = regr.predict(X_thetas_recalibration)
this_r = np.exp(prediction[:, 1])
if t == settings.theta_observed:
r_recalibration_sm = this_r
recalibration_expected_r.append(np.mean(this_r / r_recalibration_sm))
################################################################################
# Illustration
################################################################################
if t == settings.theta_benchmark_illustration:
# Prepare data for illustration
thetas0_array = np.zeros((X_illustration_transformed.shape[0], 2),
dtype=X_illustration_transformed.dtype)
thetas0_array[:, :] = settings.thetas[t]
X_thetas_illustration = np.hstack((X_illustration_transformed, thetas0_array))
# Evaluate illustration data
prediction = regr.predict(X_thetas_illustration)
r_hat_illustration = np.exp(prediction[:, 1])
np.save(results_dir + '/r_illustration_' + algorithm + filename_addition + '.npy', r_hat_illustration)
################################################################################
# Neyman construction toys
################################################################################
if do_neyman:
# Prepare alternate data for Neyman construction
thetas0_array = np.zeros((X_neyman_alternate_transformed.shape[0], 2),
dtype=X_neyman_alternate_transformed.dtype)
thetas0_array[:, :] = theta
X_thetas_neyman_alternate = np.hstack((X_neyman_alternate_transformed, thetas0_array))
# Neyman construction: evaluate alternate sample (raw)
log_r_neyman_alternate = regr.predict(X_thetas_neyman_alternate)[:, 1]
llr_neyman_alternate = -2. * np.sum(log_r_neyman_alternate.reshape((-1, n_expected_events_neyman)),
axis=1)
np.save(neyman_dir + '/' + neyman_filename + '_llr_alternate_' + str(
t) + '_' + algorithm + filename_addition + '.npy', llr_neyman_alternate)
# NC: null
X_neyman_null = np.load(settings.unweighted_events_dir + \
'/neyman/' + input_X_prefix + 'X_' + \
neyman_filename + '_null_' + str(t) + '.npy')
X_neyman_null_transformed = scaler.transform(
X_neyman_null.reshape((-1, X_neyman_null.shape[2])))
# Prepare null data for Neyman construction
thetas0_array = np.zeros((X_neyman_null_transformed.shape[0], 2),
dtype=X_neyman_null_transformed.dtype)
thetas0_array[:, :] = settings.thetas[t]
X_thetas_neyman_null = np.hstack((X_neyman_null_transformed, thetas0_array))
# Neyman construction: evaluate null sample (raw)
log_r_neyman_null = regr.predict(X_thetas_neyman_null)[:, 1]
llr_neyman_null = -2.*np.sum(log_r_neyman_null.reshape(
(-1, n_expected_events_neyman)), axis=1)
np.save(neyman_dir + '/' + neyman_filename + '_llr_null_' + \
str(t) + '_' + algorithm + filename_addition + \
'.npy', llr_neyman_null)
# NC: null evaluated at alternate
if t == settings.theta_observed:
for tt in range(settings.n_thetas):
X_neyman_null = np.load(settings.unweighted_events_dir + \
'/neyman/' + input_X_prefix + \
'X_' + neyman_filename + \
'_null_' + str(tt) + '.npy')
X_neyman_null_transformed = scaler.transform(
X_neyman_null.reshape((-1, X_neyman_null.shape[2])))
X_thetas_neyman_null = np.hstack((X_neyman_null_transformed,
thetas0_array))
# Neyman construction: evaluate null sample (raw)
log_r_neyman_null = regr.predict(X_thetas_neyman_null)[:, 1]
llr_neyman_null = -2. * np.sum(log_r_neyman_null.reshape((-1, n_expected_events_neyman)), axis=1)
np.save(neyman_dir + '/' + neyman_filename + \
'_llr_nullatalternate_' + str(tt) + '_' + \
algorithm + filename_addition + '.npy', llr_neyman_null)
# Save evaluation results
expected_llr = np.asarray(expected_llr)
mse_log_r = np.asarray(mse_log_r)
trimmed_mse_log_r = np.asarray(trimmed_mse_log_r)
expected_r_vs_sm = np.asarray(expected_r_vs_sm)
np.save(results_dir + '/llr_' + algorithm + filename_addition + '.npy', expected_llr)
np.save(results_dir + '/mse_logr_' + algorithm + filename_addition + '.npy', mse_log_r)
np.save(results_dir + '/trimmed_mse_logr_' + algorithm + filename_addition + '.npy', trimmed_mse_log_r)
np.save(results_dir + '/expected_r_vs_sm_' + algorithm + filename_addition + '.npy',
expected_r_vs_sm)
if recalibration_mode:
recalibration_expected_r = np.asarray(recalibration_expected_r)
np.save(results_dir + '/recalibration_expected_r_vs_sm_' + \
algorithm + filename_addition + '.npy',
recalibration_expected_r)
# Evaluation times
logging.info('Evaluation timing: median %s s, mean %s s',
np.median(eval_times), np.mean(eval_times))
logging.info('Starting roaming')
r_roam = []
for i in range(n_roaming):
prediction = regr.predict(X_thetas_roam[i])
r_roam.append(np.exp(prediction[:, 1]))
r_roam = np.asarray(r_roam)
np.save(results_dir + '/r_roam_' + algorithm + filename_addition + '.npy', r_roam)
################################################################################
# Calibrated evaluation loop
################################################################################
logging.info('Starting calibrated evaluation and roaming')
expected_llr_calibrated = []
mse_log_r_calibrated = []
trimmed_mse_log_r_calibrated = []
r_roam_temp = np.zeros((settings.n_thetas, n_roaming))
eval_times = []
expected_r_vs_sm = []
if recalibration_mode:
recalibration_expected_r = []
for t, theta in enumerate(settings.thetas):
if (t + 1) % 100 == 0:
logging.info('Starting theta %s / %s', t + 1, settings.n_thetas)
################################################################################
# Calibration
################################################################################
# Prepare data for calibration
n_calibration_each = X_calibration_transformed.shape[0]
thetas0_array = np.zeros((n_calibration_each, 2),
dtype=X_calibration_transformed.dtype)
thetas0_array[:, :] = settings.thetas[t]
X_thetas_calibration = np.hstack((X_calibration_transformed, thetas0_array))
X_thetas_calibration = np.vstack((X_thetas_calibration, X_thetas_calibration))
y_calibration = np.zeros(2 * n_calibration_each)
y_calibration[n_calibration_each:] = 1.
w_calibration = np.zeros(2 * n_calibration_each)
w_calibration[:n_calibration_each] = weights_calibration[t]
w_calibration[n_calibration_each:] = weights_calibration[theta1]
# Calibration
ratio_calibrated = ClassifierScoreRatio(
CalibratedClassifierScoreCV(regr, cv='prefit', method='isotonic'))
ratio_calibrated.fit(X_thetas_calibration, y_calibration,
sample_weight=w_calibration)
################################################################################
# Evaluation
################################################################################
# Prepare data
thetas0_array = np.zeros((X_test_transformed.shape[0], 2),
dtype=X_test_transformed.dtype)
thetas0_array[:, :] = settings.thetas[t]
X_thetas_test = np.hstack((X_test_transformed, thetas0_array))
time_before = time.time()
this_r, this_other = ratio_calibrated.predict(X_thetas_test)
eval_times.append(time.time() - time_before)
this_score = this_other[:, 1:3]
# Extract numbers of interest
expected_llr_calibrated.append(- 2.*settings.n_expected_events/ \
n_events_test*np.sum(np.log(this_r)))
mse_log_r_calibrated.append(
calculate_mean_squared_error(np.log(r_test[t]), np.log(this_r), 0.))
trimmed_mse_log_r_calibrated.append(
calculate_mean_squared_error(np.log(r_test[t]), np.log(this_r), 'auto'))
if t == settings.theta_observed:
r_sm = this_r
expected_r_vs_sm.append(np.mean(this_r / r_sm))
# For benchmark theta, save more data
if t == settings.theta_benchmark_nottrained:
np.save(results_dir + '/scores_nottrained_' + algorithm + '_calibrated' + filename_addition + '.npy',
this_score)
np.save(results_dir + '/r_nottrained_' + algorithm + '_calibrated' + filename_addition + '.npy', this_r)
np.save(results_dir + '/r_vs_sm_nottrained_' + algorithm + '_calibrated' + filename_addition + '.npy',
this_r / r_sm)
np.save(results_dir + '/calvalues_nottrained_' + algorithm + filename_addition + '.npy',
ratio_calibrated.classifier_.calibration_sample[:n_calibration_each])
elif t == settings.theta_benchmark_trained:
np.save(results_dir + '/scores_trained_' + algorithm + '_calibrated' + filename_addition + '.npy',
this_score)
np.save(results_dir + '/r_trained_' + algorithm + '_calibrated' + filename_addition + '.npy', this_r)
np.save(results_dir + '/r_vs_sm_trained_' + algorithm + '_calibrated' + filename_addition + '.npy',
this_r / r_sm)
np.save(results_dir + '/calvalues_trained_' + algorithm + filename_addition + '.npy',
ratio_calibrated.classifier_.calibration_sample[:n_calibration_each])
################################################################################
# Recalibration
################################################################################
if recalibration_mode:
# Prepare data for recalibration
thetas0_array = np.zeros((X_recalibration_transformed.shape[0], 2), dtype=X_recalibration_transformed.dtype)
thetas0_array[:, :] = settings.thetas[t]
X_thetas_recalibration = np.hstack((X_recalibration_transformed, thetas0_array))
# Evaluate recalibration data
this_r, _ = ratio_calibrated.predict(X_thetas_recalibration)
if t == settings.theta_observed:
r_recalibration_sm = this_r
recalibration_expected_r.append(np.mean(this_r / r_recalibration_sm))
################################################################################
# Illustration
################################################################################
if t == settings.theta_benchmark_illustration:
# Prepare data for illustration
thetas0_array = np.zeros((X_illustration_transformed.shape[0], 2),
dtype=X_illustration_transformed.dtype)
thetas0_array[:, :] = settings.thetas[t]
X_thetas_illustration = np.hstack((X_illustration_transformed, thetas0_array))
# Evaluate illustration data
r_hat_illustration, _ = ratio_calibrated.predict(X_thetas_illustration)
np.save(results_dir + '/r_illustration_' + algorithm + '_calibrated' + filename_addition + '.npy',
r_hat_illustration)
################################################################################
# Neyman construction toys
################################################################################
# Neyman construction
if do_neyman:
# Prepare alternate data for Neyman construction
thetas0_array = np.zeros((X_neyman_alternate_transformed.shape[0], 2),
dtype=X_neyman_alternate_transformed.dtype)
thetas0_array[:, :] = settings.thetas[t]
X_thetas_neyman_alternate = np.hstack((X_neyman_alternate_transformed, thetas0_array))
# Neyman construction: alternate (calibrated)
r_neyman_alternate, _ = ratio_calibrated.predict(X_thetas_neyman_alternate)
llr_neyman_alternate = -2. * np.sum(np.log(r_neyman_alternate).reshape((-1, n_expected_events_neyman)),
axis=1)
np.save(neyman_dir + '/' + neyman_filename + '_llr_alternate_' + str(
t) + '_' + algorithm + '_calibrated' + filename_addition + '.npy', llr_neyman_alternate)
# Neyman construction: null
X_neyman_null = np.load(
settings.unweighted_events_dir + '/neyman/' + input_X_prefix + 'X_' + neyman_filename + '_null_' + str(
t) + '.npy')
X_neyman_null_transformed = scaler.transform(
X_neyman_null.reshape((-1, X_neyman_null.shape[2])))
# Prepare null data for Neyman construction
thetas0_array = np.zeros((X_neyman_null_transformed.shape[0], 2),
dtype=X_neyman_null_transformed.dtype)
thetas0_array[:, :] = settings.thetas[t]
X_thetas_neyman_null = np.hstack((X_neyman_null_transformed, thetas0_array))
# Neyman construction: evaluate null (calibrated)
r_neyman_null, _ = ratio_calibrated.predict(X_thetas_neyman_null)
llr_neyman_null = -2. * np.sum(
np.log(r_neyman_null).reshape((-1, n_expected_events_neyman)), axis=1)
np.save(neyman_dir + '/' + neyman_filename + '_llr_null_' + str(
t) + '_' + algorithm + '_calibrated' + filename_addition + '.npy', llr_neyman_null)
# NC: null evaluated at alternate
if t == settings.theta_observed:
for tt in range(settings.n_thetas):
X_neyman_null = np.load(settings.unweighted_events_dir + \
'/neyman/' + input_X_prefix + \
'X_' + neyman_filename + \
'_null_' + str(tt) + '.npy')
X_neyman_null_transformed = scaler.transform(
X_neyman_null.reshape((-1, X_neyman_null.shape[2])))
X_thetas_neyman_null = np.hstack((X_neyman_null_transformed,
thetas0_array))
# Neyman construction: evaluate null sample (calibrated)
r_neyman_null, _ = ratio_calibrated.predict(X_thetas_neyman_null)
llr_neyman_null = -2.*np.sum(
np.log(r_neyman_null).reshape((-1,
n_expected_events_neyman)),
axis=1)
np.save(neyman_dir + '/' + neyman_filename + \
'_llr_nullatalternate_' + str(tt) + '_' + \
algorithm + '_calibrated' + filename_addition + \
'.npy', llr_neyman_null)
# Roaming
thetas0_array = np.zeros((n_roaming, 2), dtype=X_roam_transformed.dtype)
thetas0_array[:, :] = settings.thetas[t]
X_thetas_roaming_temp = np.hstack((X_roam_transformed, thetas0_array))
r_roam_temp[t, :], _ = ratio_calibrated.predict(X_thetas_roaming_temp)
# Save evaluation results
expected_llr_calibrated = np.asarray(expected_llr_calibrated)
mse_log_r_calibrated = np.asarray(mse_log_r_calibrated)
trimmed_mse_log_r_calibrated = np.asarray(trimmed_mse_log_r_calibrated)
expected_r_vs_sm = np.asarray(expected_r_vs_sm)
if recalibration_mode:
recalibration_expected_r = np.asarray(recalibration_expected_r)
np.save(results_dir + '/llr_' + algorithm + '_calibrated' + \
filename_addition + '.npy', expected_llr_calibrated)
np.save(results_dir + '/mse_logr_' + algorithm + '_calibrated' + \
filename_addition + '.npy', mse_log_r_calibrated)
np.save(results_dir + '/trimmed_mse_logr_' + algorithm + \
'_calibrated' + filename_addition + '.npy', trimmed_mse_log_r_calibrated)
np.save(results_dir + '/expected_r_vs_sm_' + algorithm + \
'_calibrated' + filename_addition + '.npy', expected_r_vs_sm)
if recalibration_mode:
recalibration_expected_r = np.asarray(recalibration_expected_r)
np.save(results_dir + '/recalibration_expected_r_vs_sm_' + \
algorithm + '_calibrated' + filename_addition + '.npy',
recalibration_expected_r)
# Evaluation times
logging.info('Calibrated evaluation timing: median %s s, mean %s s',
np.median(eval_times), np.mean(eval_times))
logging.info('Interpolating calibrated roaming')
gp = GaussianProcessRegressor(normalize_y=True,
kernel=C(1.0) * Matern(1.0, nu=0.5),
n_restarts_optimizer=10)
gp.fit(settings.thetas[:], np.log(r_roam_temp))
r_roam_calibrated = np.exp(gp.predict(np.c_[xx.ravel(), yy.ravel()])).T
np.save(results_dir + '/r_roam_' + algorithm + '_calibrated' + \
filename_addition + '.npy', r_roam_calibrated)
def run(args, mass, winLow, winHigh, bkghist, template, optKernel):
GPh = GPHisto(bkghist)
GPh.setWindow(winLow, winHigh)
X = GPh.getXWindowArr()
y = GPh.getYWindowArr()
dy = GPh.getErrWindowArr()
# The distributions with no window removed.
X_t = GPh.getXArr()
y_t = GPh.getYArr()
dy_t = GPh.getErrArr()
if args.noWindow:
X = X_t
y = y_t
dy = dy_t
x = GPh.getXArr()
gp = None
if optKernel is None:
kernel = C(10.0, (1e-3, 1e15)) * RBF(
50.0, (1e-3, 1e6)) #squared exponential kernel
#kernel = C(1000.0, (1e-3, 1e15)) * FallExp(1.0, (1e-5, 1e2), 1.0, (1e-3,1e15)) * Gibbs(1.0, (1e-3, 1e5), 1.0, (1e-3,1e5))
gp = GaussianProcessRegressor(
kernel=kernel
#,optimizer=None
,
alpha=dy**2,
n_restarts_optimizer=9)
else:
kernel = optKernel
gp = GaussianProcessRegressor(kernel=kernel,
optimizer=None,
alpha=dy**2,
n_restarts_optimizer=9)
gp.fit(X, y)
y_pred, sigma = gp.predict(x, return_std=True)
outhist = GPh.getHisto(y_pred, 1.96 * sigma, 'GP Fit')
print "outhist nbins:", outhist.GetNbinsX()
if args.noWindow:
bkgWindow = GPh.getHisto(y, dy, 'Full Background')
else:
bkgWindow = GPh.getWinHisto(y, dy, 'Full Background')
errCov = outhist.Clone('errorCoverage')
errCov.Reset()
norm = (bkghist.Integral() / template.Integral())
print gp.kernel_
for nbin in range(1, outhist.GetNbinsX()):
Err = outhist.GetBinError(nbin)
cont = outhist.GetBinContent(nbin)
binCenter = outhist.GetBinCenter(nbin)
temp = norm * template.GetBinContent(template.FindBin(binCenter))
if temp <= (cont + Err) and temp >= (cont - Err):
errCov.Fill(binCenter)
else:
print "found uncovered bin: ", nbin
print "LowErr : template : HighErr ---- {0} : {1} : {2}".format(
(cont - Err), temp, (cont + Err))
print "----------"
return errCov
pass #run
import numpy as np
from scipy.optimize import fmin_l_bfgs_b
from scipy.stats import uniform
import matplotlib.pyplot as plt
from sklearn.gaussian_process.kernels import Matern, ConstantKernel as C
from bolero.environment.catapult import Catapult
from bolero_bayes_opt import BOPSOptimizer
catapult = Catapult([(0, 0), (2.0, -0.5), (3.0, 0.5), (4, 0.25), (5, 2.5),
(7, 0), (10, 0.5), (15, 0)])
catapult.init()
kernel = C(100.0, (1.0, 10000.0)) \
* Matern(length_scale=(1.0, 1.0), length_scale_bounds=[(0.1, 100), (0.1, 100)])
opt = BOPSOptimizer(boundaries=[(5, 10), (0, np.pi / 2)],
bo_type="bo",
acquisition_function="UCB",
acq_fct_kwargs=dict(kappa=2.5),
optimizer="direct+lbfgs",
maxf=100,
gp_kwargs=dict(kernel=kernel, normalize_y=True,
alpha=1e-5))
target = 4.0 # Fixed target
context = (target - 2.0) / 8.0
# Compute maximal achievable reward for this target
optimal_reward = -np.inf
def gomez_bombarelli_opt(X_train,
pred_vae,
ground_truth,
total_it=10000,
constrained=True):
"""Runs the Gomez-Bombarelli optimization algorithm"""
LD = 20
L = X_train.shape[1]
embeddings = pred_vae.encoder_.predict([X_train])[0]
predictions = pred_vae.predictor_.predict(embeddings)[0]
predictions = predictions.reshape((predictions.shape[0]))
kernel = C(1.0, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2))
gp = GaussianProcessRegressor(kernel=kernel,
n_restarts_optimizer=0,
normalize_y=True)
print("Fitting GP...")
gp.fit(embeddings, predictions)
print("Finishing fitting GP")
def gp_z_estimator(z):
val = gp.predict(z.reshape(1, -1)).ravel()[0]
return -val
method = 'COBYLA'
num_it = 0
f_opts = []
gt_opts = []
z0 = np.random.randn(LD)
if constrained:
z_norm = np.linalg.norm(embeddings, axis=1)
z_norm_mean = np.mean(z_norm)
z_norm_std = np.std(z_norm)
lower_rad = z_norm_mean - z_norm_std * 2
higher_rad = z_norm_mean + z_norm_std * 2
constraints = [{
'type': 'ineq',
'fun': lambda x: np.linalg.norm(x) - lower_rad
}, {
'type': 'ineq',
'fun': lambda x: higher_rad - np.linalg.norm(x)
}]
res = minimize(gp_z_estimator,
z0,
method=method,
constraints=constraints,
options={
'disp': False,
'tol': 0.1,
'maxiter': 10000
},
tol=0.1)
else:
res = minimize(gp_z_estimator,
z0,
method=method,
options={
'disp': False,
'tol': 0.1,
'maxiter': 10000
},
tol=0.1)
z_opt = res.x.reshape((1, LD))
f_opt = -res.fun
x_opt = np.argmax(pred_vae.decoder_.predict(z_opt), axis=-1)
gt_opt = ground_truth.predict(x_opt)[:, 0][0]
print(f_opt, gt_opt)
oracle_max_seq = util.convert_idx_array_to_aas(x_opt)
max_dict = {
'oracle_max': f_opt,
'oracle_max_seq': oracle_max_seq,
'gt_of_oracle_max': gt_opt
}
results = np.array([f_opt, gt_opt])
return results, max_dict
def main(self):
self.x = [-4, 4, 5]
self.y = [-4, 4, 5]
self.xy, shape1 = self.create_parameter_space([self.x, self.y])
xy0 = self.xy # Save initial grid for sampling
z = []
for i in range(np.shape(self.xy)[0]):
z.append(self.f(self.xy[i][0], self.xy[i][1]))
self.z = np.array(z)
# print(self.parameter_space_shape)
# print(xy[:,0])
# print(xy[0,:])
# print(np.shape(xy))
# print(np.shape(z))
# Add noise:
dz = 0.01 + 0.02 * np.random.random(self.z.shape)
noise = np.random.normal(0, dz)
self.z += noise
# Instantiate a Gaussian Process model
kernel = C(1.0, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2)) + WhiteKernel(
) # This seems to work and is what Robbie suggested
# kernel = DotProduct() + WhiteKernel()
# kernel = 1.0 * RBF(1.0) + 1.0 * WhiteKernel() # this doesn't give an equivalent output to None. Not sure why.
# kernel = None
self.gp = GaussianProcessRegressor(
kernel=kernel, n_restarts_optimizer=9
) # None is passed, the kernel “1.0 * RBF(1.0)” is used as default.
# Fit to data using Maximum Likelihood Estimation of the parameters
print("xy: ", np.shape(self.xy))
print("z: ", np.shape(self.z))
self.gp.fit(self.xy, self.z)
# Make the prediction on the meshed x-axis (ask for MSE as well)
z_preds, sigma = self.gp.predict(self.xy, return_std=True)
# self.plot(self.z, z_preds, self.xy, shape1, plot_residuals = True)
self.calculate_avg_residual(self.z, z_preds)
print("Initial min: %f" % min(z_preds))
xx = [-4, 4, 15]
yy = [-4, 4, 15]
xy_dense, shape2 = self.create_parameter_space([xx, yy])
zdense = []
for i in range(np.shape(xy_dense)[0]):
zdense.append(self.f(xy_dense[i][0], xy_dense[i][1]))
zdense = np.array(zdense)
zdense_preds, sigma = self.gp.predict(xy_dense, return_std=True)
# self.plot(zdense, zdense_preds, xy_dense, shape2, plot_residuals = True)
self.calculate_avg_residual(zdense, zdense_preds)
print("Initial min on dense mesh: %f" % min(zdense_preds))
self.min_search()
zdense_preds, sigma = self.gp.predict(
xy_dense, return_std=True) # Replot things after the minsearch
# self.plot(zdense, zdense_preds, xy_dense, shape2, plot_residuals = True)
self.calculate_avg_residual(zdense, zdense_preds)
print("Min on dense mesh after training: %f" % min(zdense_preds))
self.min_search()
zdense_preds, sigma = self.gp.predict(
xy_dense, return_std=True) # Replot things after the minsearch
# self.plot(zdense, zdense_preds, xy_dense, shape2, plot_residuals = True)
self.calculate_avg_residual(zdense, zdense_preds)
print("Min on dense mesh after training: %f" % min(zdense_preds))
self.min_search()
zdense_preds, sigma = self.gp.predict(
xy_dense, return_std=True) # Replot things after the minsearch
# self.plot(zdense, zdense_preds, xy_dense, shape2, plot_residuals = True)
self.calculate_avg_residual(zdense, zdense_preds)
print("Min on dense mesh after training: %f" % min(zdense_preds))
samplePoints = 10000
#how many random measurements of the function will be done before calculating expected improvement
no_startingPoints = 100
#
tavalist = 100
suurem = 10000
#algpunktide genereerimine
X = np.random.uniform(bounds[0, 0], bounds[0, 1], [no_startingPoints, 1])
Y = np.random.uniform(bounds[1, 0], bounds[1, 1], [no_startingPoints, 1])
XY = np.column_stack((X, Y))
Z = tf.rosenbrock(XY, 1, 100)
#Mis mudel ja Kernel
customKernel = C(1.0) * Matern()
model = GaussianProcessRegressor(kernel=customKernel)
for i in range(n_iterations):
#print iteration
print("Iteration " + str(i) + "/" + str(n_iterations))
# Update Gaussian process with existing samples
model.fit(XY, Z)
# Obtain next sampling point from the acquisition function (expected_improvement)
XY_next = bf.propose_location(bf.expected_improvement, XY, Z, model,
bounds, samplePoints, tavalist, suurem,
no_startingPoints, exploration)
# Obtain next sample from the objective function
# ### 4. Busca los mejores hiperparámetros del GP para predecir la serie temporal del CO2 usando datos hasta la fecha más reciente. Compara estos hiperparámetros con los que se encontraron al usar datos hasta diciembre de 2001 (los datos usados en la práctica).
# In[6]:
from sklearn.gaussian_process.kernels import RationalQuadratic
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C, WhiteKernel, ExpSineSquared
from time import time
# Kernel with generic parameters
k1 = C(50, (1e1, 1e+3)) * RBF(50,(1e0,1e5)) # long term smooth rising trend
k2 = C(5, (1e1, 1e+2)) * RBF(100,(1e0,1e5)) * ExpSineSquared(length_scale=1.0, length_scale_bounds=(1e-1,1e1),
periodicity=1.0, periodicity_bounds="fixed") # seasonal component
k3 = C(0.5, (1e-2, 1e1)) * RationalQuadratic(length_scale=1.0, length_scale_bounds=(1e-2,1e3),
alpha=1.0, alpha_bounds=(1e-1,1e3)) # medium term irregularities
k4 = C(0.1, (1e-3, 1e+1)) * RBF(0.1,(1e-2,1e2)) + WhiteKernel(noise_level=0.1**2,noise_level_bounds=(1e-4, 1e0)) # noise terms
kernel = k1 + k2 + k3 + k4
# Fit the Gaussian Process regressor, and optimize the hyperparameters
gp = GaussianProcessRegressor(kernel=kernel, alpha=0,
normalize_y=True, n_restarts_optimizer=1)
t0 = time()
# In[7]:
ystar, sigma = gpr.predict(x, return_std=True)
theta = gpr.kernel_.theta
#lml = gpr.log_marginal_likelihood(theta=theta)
lml = gpr.log_marginal_likelihood()
sigma = np.reshape(sigma, (np.size(sigma), 1))
sigma = (sigma**2 + 1)**0.5
ystarp = ystar + sigma
ystari = scaler.inverse_transform(ystar)
ystarpi = scaler.inverse_transform(ystarp)
sigmai = (ystarpi - ystari)
return ystari, sigmai, gpr, theta, lml
k1_iwu = np.random.uniform(1e-1, 1e1)
k2_iwu = np.random.uniform(1e-1, 1e1)
kernel_standard_gp = C(k1_iwu, (1e-1, 1e1)) * RBF(k2_iwu, (1e-1, 1e1))
start = time.time()
ypred_sk_weak, sigma_sk_weak, _, theta, lml = GP_train_kernel(
x, snr, kernel_standard_gp)
end = time.time()
#np.savetxt('ypred_sk_weak.csv', ypred_sk, delimiter=',')
#np.savetxt('sigma_sk_weak.csv', sigma_sk, delimiter=',')
print("weak prior")
print("GP fitting took " + str((end - start) / 60) + " minutes")
print("LML = " + str(lml))
thetatrans = np.exp(theta)
print("theta (linear) = " + str(thetatrans))
print("theta (log) = " + str(theta))
print("sigma mean = " + str(np.mean(sigma_sk_weak)))
kernel_standard_gp = C(k1_o, (k1_o / 1000, k1_o * 1000)) * RBF(
def GP(lc, restarts=0, kernel_type=1):
bands = np.unique(lc.photometry['lambda_cen'])
X = np.vstack([
lc.photometry['mjd'] - np.min(lc.photometry['mjd']),
lc.photometry['lambda_cen']
]).T
y = lc.photometry['flux'] - np.min(lc.photometry['flux'])
dy = lc.photometry['fluxerr']
if kernel_type == 0: ### Fit a 9-parameter (3x3 RBF) kernel
kn = kernel_RBF()
final_gp = GaussianProcessRegressor(kernel=kn,
alpha=dy,
n_restarts_optimizer=restarts)
final_gp.fit(X, y)
kps = final_gp.kernel_.get_params()
kernel_params = [kps['k1__k1__k1'].constant_value,\
kps['k1__k1__k2'].length_scale[0],\
kps['k1__k1__k2'].length_scale[1],\
kps['k1__k2__k1'].constant_value,\
kps['k1__k2__k2'].length_scale[0],\
kps['k1__k2__k2'].length_scale[1],\
kps['k2__k1'].constant_value,\
kps['k2__k2'].length_scale[0],\
kps['k2__k2'].length_scale[1],\
final_gp.log_marginal_likelihood(),\
]
elif kernel_type == 1: ### Iteratively fit to the residuals
k1 = kernel_RBF_general(5e3, 6e3, 1e0, 1e6, 1e0, 1e6)
gp1 = GaussianProcessRegressor(kernel=k1,
alpha=dy,
n_restarts_optimizer=restarts)
gp1.fit(X, y)
kp1 = gp1.kernel_.get_params()
gp1_result = gp1.predict(X)
### Fit the residuals to an RBF with smaller length scales
k2 = kernel_RBF_general(1e2, 3e2, 1e0, kp1['length_scale'][0], 1e0,
kp1['length_scale'][1])
gp2 = GaussianProcessRegressor(kernel=k2,
alpha=dy,
n_restarts_optimizer=restarts)
gp2.fit(X, y - gp1_result)
kp2 = gp2.kernel_.get_params()
gp2_result = gp2.predict(X)
### Fit the second residuals to a final RBF with the smallest length scales (trying to capture noise).
k3 = kernel_RBF_general(5e0, 3e1, 1e0, kp2['length_scale'][0], 1e0,
kp2['length_scale'][1])
gp3 = GaussianProcessRegressor(kernel=k3,
alpha=dy,
n_restarts_optimizer=restarts)
gp3.fit(X, y - gp1_result - gp2_result)
kp3 = gp3.kernel_.get_params()
final_kernel = C(1.,(1e-2,1e3)) * RBF((kp1['length_scale'][0],kp1['length_scale'][1])) +\
C(1.,(1e-2,1e3)) * RBF((kp2['length_scale'][0],kp2['length_scale'][1])) +\
C(1.,(1e-2,1e3)) * RBF((kp3['length_scale'][0],kp3['length_scale'][1]))
final_gp = GaussianProcessRegressor(kernel=final_kernel,
alpha=dy,
n_restarts_optimizer=restarts)
final_gp.fit(X, y)
kps = final_gp.kernel_.get_params()
kernel_params = [kps['k1__k1__k1'].constant_value,\
kps['k1__k1__k2'].length_scale[0],\
kps['k1__k1__k2'].length_scale[1],\
kps['k1__k2__k1'].constant_value,\
kps['k1__k2__k2'].length_scale[0],\
kps['k1__k2__k2'].length_scale[1],\
kps['k2__k1'].constant_value,\
kps['k2__k2'].length_scale[0],\
kps['k2__k2'].length_scale[1],\
final_gp.log_marginal_likelihood(),\
]
elif kernel_type == 2:
kn = kernel_Matern()
final_gp = GaussianProcessRegressor(kernel=kn,
alpha=dy,
n_restarts_optimizer=restarts)
final_gp.fit(X, y)
kps = final_gp.kernel_.get_params()
kernel_params = [kps['k1'].constant_value,\
kps['k2'].length_scale[0],\
kps['k2'].length_scale[1],\
final_gp.log_marginal_likelihood(),\
]
else:
raise ValueError('Parameter "kernel_type" must be one of [0,1,2].')
# generate realization of GP fit on the data
time_real = np.arange(np.floor(np.min(X[:, 0])),
np.ceil(np.max(X[:, 0])) + 1.)
tgp, wgp = np.meshgrid(time_real, bands)
x_real = np.vstack([tgp.flatten(), wgp.flatten()]).T
y_real = final_gp.predict(x_real)
y_real = y_real.reshape(tgp.shape).T
# compute features from the realization of the fit
# Peak magnitude of the GP flux prediction in the LSST i band
pkmag_i = 22.0 - 2.5 * np.log10(np.max(y_real[:, 3]))
# Ratio of the maximum positive flux to the maximum-minus-minimum flux in the LSST i band
pos_flux_ratio = np.max(
y_real[:, 3]) / (np.max(y_real[:, 3]) - np.min(y_real[:, 3]))
# Normalized difference of the light curve colors at maximum/minimum light. The blue
# measurement is the difference between the LSST i and g bands, and the red measurement
# is the difference between the LSST y and i bands. The normalized difference is calculated
# by taking the difference of the fluxes in the two bands divided by their sum.
max_light_idx = np.where(y_real == np.max(y_real))[0][0]
min_light_idx = np.where(y_real == np.min(y_real))[0][0]
#
max_fr_blue = (y_real[max_light_idx, 3] - y_real[max_light_idx, 1]) / (
y_real[max_light_idx, 3] + y_real[max_light_idx, 1])
min_fr_blue = (y_real[min_light_idx, 3] - y_real[min_light_idx, 1]) / (
y_real[min_light_idx, 3] + y_real[min_light_idx, 1])
max_fr_red = (y_real[max_light_idx, 5] - y_real[max_light_idx, 3]) / (
y_real[max_light_idx, 5] + y_real[max_light_idx, 3])
min_fr_red = (y_real[min_light_idx, 5] - y_real[min_light_idx, 3]) / (
y_real[min_light_idx, 5] + y_real[min_light_idx, 3])
# Difference of the time of maximum in the LSST y and g bands in days
max_dt_yg = time_real[np.argmax(y_real[:, 5])] - time_real[np.argmax(
y_real[:, 1])]
# An estimate of the light curve “width” that is applicable even for non-supernova-like
# transients and variables. This is implemented as the integral of the positive/negative
# parts of the GP flux predictions divided by the positive/negative maximum fluxes.
positive_width = integrate.trapz(y_real[:, 3] -
np.median(y_real[:, 3])) / np.max(
y_real[:, 3])
# Measurements of the rise and decline times of a light curve. This measurement is defined
# as the time in days for the light curve to rise (bwd) or decline (fwd) to a given fraction
# (either 20% or 50%) of maximum light in the LSST i band.
### NOTE: REVISIT THE DEFAULT EXCEPTION VALUES HERE (taken to be 1)
### REPLACE WITH MEAN VALUES
try:
time_fwd_max_20 = np.where(np.logical_and(\
y_real[:,3] <= (0.2 * (np.max(y_real[:,3]) - np.min(y_real[:,3])) + np.min(y_real[:,3])),\
time_real > np.argmax(y_real[:,3])\
))[0][0] - np.argmax(y_real[:,3])
except IndexError:
time_fwd_max_20 = 1.
try:
time_fwd_max_50 = np.where(np.logical_and(\
y_real[:,3] <= (0.5 * (np.max(y_real[:,3]) - np.min(y_real[:,3])) + np.min(y_real[:,3])),\
time_real > np.argmax(y_real[:,3])\
))[0][0] - np.argmax(y_real[:,3])
except IndexError:
time_fwd_max_50 = 1.
try:
time_bwd_max_20 = np.argmax(y_real[:,3]) - np.where(np.logical_and(\
y_real[:,3] <= (0.2 * (np.max(y_real[:,3]) - np.min(y_real[:,3])) + np.min(y_real[:,3])),\
time_real < np.argmax(y_real[:,3])\
))[0][-1]
except IndexError:
time_bwd_max_20 = 1.
try:
time_bwd_max_50 = np.argmax(y_real[:,3]) - np.where(np.logical_and(\
y_real[:,3] <= (0.5 * (np.max(y_real[:,3]) - np.min(y_real[:,3])) + np.min(y_real[:,3])),\
time_real < np.argmax(y_real[:,3])\
))[0][-1]
except IndexError:
time_bwd_max_50 = 1.
# Ratio of the rise/decline times calculated as described above in different bands. The blue
# measurement is the difference between the LSST i and g bands, and the red measurement
# is the difference between the LSST y and i bands.
try:
time_fwd_max_20_ratio_blue = float(time_fwd_max_20 / ((np.where(
np.logical_and(
y_real[:, 1] <=
(0.2 * (np.max(y_real[:, 1]) - np.min(y_real[:, 1])) +
np.min(y_real[:, 1])), time_real > np.argmax(y_real[:, 1])))
[0][0])) -
np.argmax(y_real[:, 1]))
except IndexError:
time_fwd_max_20_ratio_blue = 1. #float(time_fwd_max_20 / int(time_real[-1]))
try:
time_fwd_max_50_ratio_blue = float(time_fwd_max_50 / ((np.where(
np.logical_and(
y_real[:, 1] <=
(0.5 * (np.max(y_real[:, 1]) - np.min(y_real[:, 1])) +
np.min(y_real[:, 1])), time_real > np.argmax(y_real[:, 1])))
[0][0])) -
np.argmax(y_real[:, 1]))
except IndexError:
time_fwd_max_50_ratio_blue = 1. #float(time_fwd_max_50 / int(time_real[-1]))
try:
time_bwd_max_20_ratio_blue = float(
time_bwd_max_20 / (np.argmax(y_real[:, 1]) - (np.where(
np.logical_and(
y_real[:, 1] <=
(0.2 * (np.max(y_real[:, 1]) - np.min(y_real[:, 1])) +
np.min(y_real[:, 1])),
time_real < np.argmax(y_real[:, 1])))[0][-1])))
except IndexError:
time_bwd_max_20_ratio_blue = 1. #float(time_bwd_max_20 / int(time_real[1]))
try:
time_bwd_max_50_ratio_blue = float(
time_bwd_max_50 / (np.argmax(y_real[:, 1]) - (np.where(
np.logical_and(
y_real[:, 1] <=
(0.5 * (np.max(y_real[:, 1]) - np.min(y_real[:, 1])) +
np.min(y_real[:, 1])),
time_real < np.argmax(y_real[:, 1])))[0][-1])))
except IndexError:
time_bwd_max_50_ratio_blue = 1. #float(time_bwd_max_50 / int(time_real[1]))
try:
time_fwd_max_20_ratio_red = float(((np.where(
np.logical_and(
y_real[:, 5] <=
(0.2 * (np.max(y_real[:, 5]) - np.min(y_real[:, 5])) +
np.min(y_real[:, 5])), time_real > np.argmax(
y_real[:, 1])))[0][0]) - np.argmax(y_real[:, 5])) /
time_fwd_max_20)
except IndexError:
time_fwd_max_20_ratio_red = 1. #float(int(time_real[-1]) / time_fwd_max_20)
try:
time_fwd_max_50_ratio_red = float(((np.where(
np.logical_and(
y_real[:, 5] <=
(0.5 * (np.max(y_real[:, 5]) - np.min(y_real[:, 5])) +
np.min(y_real[:, 5])), time_real > np.argmax(
y_real[:, 5])))[0][0]) - np.argmax(y_real[:, 5])) /
time_fwd_max_50)
except IndexError:
time_fwd_max_50_ratio_red = 1. #float(int(time_real[-1]) / time_fwd_max_50)
try:
time_bwd_max_20_ratio_red = float((np.argmax(y_real[:, 5]) - (np.where(
np.logical_and(
y_real[:, 5] <=
(0.2 * (np.max(y_real[:, 5]) - np.min(y_real[:, 5])) +
np.min(y_real[:, 5])),
time_real < np.argmax(y_real[:, 5])))[0][-1])) /
time_bwd_max_20)
except IndexError:
time_bwd_max_20_ratio_red = 1. #float(int(time_real[0]) / time_bwd_max_20)
try:
time_bwd_max_50_ratio_red = float((np.argmax(y_real[:, 5]) - (np.where(
np.logical_and(
y_real[:, 5] <=
(0.5 * (np.max(y_real[:, 5]) - np.min(y_real[:, 5])) +
np.min(y_real[:, 5])),
time_real < np.argmax(y_real[:, 5])))[0][-1])) /
time_bwd_max_50)
except IndexError:
time_bwd_max_50_ratio_red = 1. #float(int(time_real[0]) / time_bwd_max_50)
# Fraction of observations that have a signal greater than 5/less than -5 times the noise level.
frac_s2n_5 = len(np.where(lc.photometry['SNR'] > 5.)[0]) / len(
lc.photometry['SNR'])
# Fraction of observations that have an absolute signal-to-noise less than 3
frac_background = len(np.where(
np.abs(lc.photometry['SNR']) < 3.)[0]) / len(lc.photometry['SNR'])
# Time difference in days between the first observation with a signal-to-noise greater than 5
# and the last such observation (in any band)
time_width_s2n_5 = lc.photometry['mjd'][np.where(
lc.photometry['SNR'] > 5.)[0][-1]] - lc.photometry['mjd'][np.where(
lc.photometry['SNR'] > 5.)[0][0]]
# Number of observations in any band within 5 days of maximum light
max_light_mjd = np.min(lc.photometry['mjd']) + max_light_idx
#
count_max_center = len(
np.where(
np.logical_and(lc.photometry['mjd'] < 5. + max_light_mjd,
lc.photometry['mjd'] > max_light_mjd - 5.))[0])
# Number of observations in any band between 20, 50, or 100 days before maximum light
# and 5 days after maximum light.
count_max_rise_20 = len(
np.where(
np.logical_and(lc.photometry['mjd'] < 5. + max_light_mjd,
lc.photometry['mjd'] > max_light_mjd - 20.))[0])
count_max_rise_50 = len(
np.where(
np.logical_and(lc.photometry['mjd'] < 5. + max_light_mjd,
lc.photometry['mjd'] > max_light_mjd - 50.))[0])
count_max_rise_100 = len(
np.where(
np.logical_and(lc.photometry['mjd'] < 5. + max_light_mjd,
lc.photometry['mjd'] > max_light_mjd - 100.))[0])
# Number of observations in any band between 5 days before maximum light and 20, 50, or
# 100 days after maximum light.
count_max_fall_20 = len(
np.where(
np.logical_and(lc.photometry['mjd'] < 20. + max_light_mjd,
lc.photometry['mjd'] > max_light_mjd - 5.))[0])
count_max_fall_50 = len(
np.where(
np.logical_and(lc.photometry['mjd'] < 50. + max_light_mjd,
lc.photometry['mjd'] > max_light_mjd - 5.))[0])
count_max_fall_100 = len(
np.where(
np.logical_and(lc.photometry['mjd'] < 100. + max_light_mjd,
lc.photometry['mjd'] > max_light_mjd - 5.))[0])
# Total signal-to-noise of all observations of the object.
total_s2n = np.sum(lc.photometry['SNR'])
fit_params = [pkmag_i, pos_flux_ratio, max_fr_blue, min_fr_blue, max_fr_red, min_fr_red, max_dt_yg, positive_width,\
time_fwd_max_20, time_fwd_max_50, time_bwd_max_20, time_bwd_max_50, time_fwd_max_20_ratio_blue,\
time_fwd_max_50_ratio_blue, time_bwd_max_20_ratio_blue, time_bwd_max_50_ratio_blue,\
time_fwd_max_20_ratio_red, time_fwd_max_50_ratio_red, time_bwd_max_20_ratio_red,\
time_bwd_max_50_ratio_red, frac_s2n_5, frac_background, time_width_s2n_5,\
count_max_center, count_max_rise_20, count_max_rise_50, count_max_rise_100,\
count_max_fall_20, count_max_fall_50, count_max_fall_100,\
total_s2n]
return kernel_params, fit_params
num_of_episodes = 30
q = {}
states = [(x, y) for x in range(-GRID, GRID + 1)
for y in range(-GRID, GRID + 1)]
actions = [(1, 0), (-1, 0), (0, 1), (0, -1)]
term_state = (GRID, GRID)
# train_data_f = [[2, 3], [5, 6]]
# train_data_t = [[7], [8]]
# mean = np.zeros(train_data_f.shape)
kernel = C(10.0,
(1e-3, 1e3)) * RBF(0.5,
(1e-2, 1e2)) + WhiteKernel(0.01, (1e-5, 1e5))
# sampled = np.random.multivariate_normal( np.squeeze(mean), kernel(train_data_f))
# gp.fit(train_data_f, train_data_t)
# plt.plot(data, np.sin(data))
# plt.show()
# print gp.get_params(deep=True)
# x = np.arange(-10 , 10, 0.2)[:, np.newaxis]
# print y_pred
# plt.scatter(train_data_f, train_data_t)
def test_gpr_rbf_unfitted(self):
se = (C(1.0, (1e-3, 1e3)) *
RBF(length_scale=10, length_scale_bounds=(1e-3, 1e3)))
kernel = (Sum(
se,
C(0.1, (1e-3, 1e3)) *
RBF(length_scale=1, length_scale_bounds=(1e-3, 1e3))))
gp = GaussianProcessRegressor(alpha=1e-7,
kernel=kernel,
n_restarts_optimizer=15,
normalize_y=True)
# return_cov=False, return_std=False
model_onnx = to_onnx(gp,
initial_types=[('X', FloatTensorType([]))],
dtype=np.float32,
target_opset=_TARGET_OPSET_)
self.assertTrue(model_onnx is not None)
dump_data_and_model(Xtest_.astype(np.float32),
gp,
model_onnx,
verbose=False,
basename="SklearnGaussianProcessRBFUnfitted")
# return_cov=True, return_std=True
options = {
GaussianProcessRegressor: {
"return_std": True,
"return_cov": True
}
}
try:
to_onnx(gp,
Xtrain_.astype(np.float32),
options=options,
target_opset=TARGET_OPSET)
except RuntimeError as e:
assert "Not returning standard deviation" in str(e)
# return_std=True
options = {GaussianProcessRegressor: {"return_std": True}}
model_onnx = to_onnx(gp,
options=options,
initial_types=[('X', FloatTensorType([None,
None]))],
dtype=np.float32,
target_opset=TARGET_OPSET)
self.assertTrue(model_onnx is not None)
self.check_outputs(
gp,
model_onnx,
Xtest_.astype(np.float32),
predict_attributes=options[GaussianProcessRegressor])
# return_cov=True
options = {GaussianProcessRegressor: {"return_cov": True}}
# model_onnx = to_onnx(gp, Xtrain_.astype(np.float32), options=options)
model_onnx = to_onnx(gp,
options=options,
initial_types=[('X', FloatTensorType([None,
None]))],
dtype=np.float32,
target_opset=TARGET_OPSET)
self.assertTrue(model_onnx is not None)
self.check_outputs(
gp,
model_onnx,
Xtest_.astype(np.float32),
predict_attributes=options[GaussianProcessRegressor])
"""The function to predict (classification will then consist in predicting
whether g(x) <= 0 or not)"""
return 5. - x[:, 1] - .5 * x[:, 0]**2.
# Design of experiments
X = np.array([[-4.61611719, -6.00099547], [4.10469096, 5.32782448],
[0.00000000, -0.50000000], [-6.17289014, -4.6984743],
[1.3109306, -6.93271427], [-5.03823144, 3.10584743],
[-2.87600388, 6.74310541], [5.21301203, 4.26386883]])
# Observations
y = np.array(g(X) > 0, dtype=int)
# Instanciate and fit Gaussian Process Model
kernel = C(0.1, (1e-5, np.inf)) * DotProduct(sigma_0=0.1)**2
gp = GaussianProcessClassifier(kernel=kernel)
gp.fit(X, y)
print("Learned kernel: %s " % gp.kernel_)
# Evaluate real function and the predicted probability
res = 50
x1, x2 = np.meshgrid(np.linspace(-lim, lim, res), np.linspace(-lim, lim, res))
xx = np.vstack([x1.reshape(x1.size), x2.reshape(x2.size)]).T
y_true = g(xx)
y_prob = gp.predict_proba(xx)[:, 1]
y_true = y_true.reshape((res, res))
y_prob = y_prob.reshape((res, res))
# Plot the probabilistic classification iso-values
# save history to a csv file
if save_history:
history_x = self.res.x_iters[0:self.n_calls]
if self.flag_maximize:
history_y = self.res.func_vals[0:self.n_calls] * (-1)
else:
history_y = self.res.func_vals[0:self.n_calls]
history_y_list = history_y.tolist()
history_x_list = history_x
with open(self.save_path + "SoranoSat_History_{0}.csv".format(self.date), "w") as f:
f.write(columns_name_list_str)
for X, Y in zip(history_x_list, history_y_list):
X.append(Y)
buf_list = list(map(str, X))
buf_str = ','.join(buf_list)
f.write(buf_str + '\n')
print("***** Optimization finished ({0}) *****".format(str(self.idx)))
if __name__ == "__main__":
_data_path = "data/SoranoSat_Data.csv"
iteration = 3
for i in range(iteration):
BO = BayesianOptimization(data_path=_data_path, n_calls=10, idx=i)
BO.preprocess()
_kernel = C(1.0, (1e-2, 1e2)) * RBF(1.0, (1e-2, 1e2)) + Wh(0.01, (1e-2, 1e2))
BO.gaussian_process(kernel=_kernel, save_model=True)
BO.run_optimization()
BO.save_result(save_history=False)
cut_table(data_path=_data_path, line_to_cut_off=iteration)
bkghist_template = f.Get('hmgg_c0')
template = f.Get('hmgg_c0')
bkghist_template.Rebin(8)
stats = 60000
optKernel = None
GPh = GPHisto(bkghist_template)
#GPh.setWindow(winLow,winHigh)
# The distributions with no window removed.
X = GPh.getXArr()
y = GPh.getYArr()
dy = GPh.getErrArr()
x = GPh.getXArr()
kernel = C(10.0,
(1e-3, 1e15)) * RBF(50.0,
(1e-3, 1e6)) #squared exponential kernel
#kernel = C(1000.0, (1e-3, 1e15)) * FallExp(1.0, (1e-5, 1e2), 1.0, (1e-3,1e15)) * Gibbs(1.0, (1e-3, 1e5), 1.0, (1e-3,1e5))
gp = GaussianProcessRegressor(
kernel=kernel
#,optimizer=None
,
alpha=dy**2,
n_restarts_optimizer=9)
gp.fit(X, y)
print gp.kernel_
raise ()
optKernel = gp.kernel_
def train(regress_type='hybrid', seed=1, **kwargs):
X_obs = kwargs['X_obs']
y_obs = kwargs['y_obs']
Kds = kwargs['Kds']
kwargs['regress_type'] = regress_type
# Debug.
#X_obs = X_obs[:10]
#y_obs = y_obs[:10]
# Fit the model.
if regress_type == 'baseline':
from baseline import Baseline
regressor = Baseline()
elif regress_type == 'mlper1':
regressor = mlp_ensemble(
n_neurons=200,
n_regressors=1,
n_epochs=50,
seed=seed,
)
elif regress_type == 'dmlper1':
regressor = mlp_ensemble(
n_neurons=1000,
n_regressors=1,
n_hidden_layers=15,
n_epochs=300,
seed=seed,
)
elif regress_type == 'mlper1g':
regressor = mlp_ensemble(
n_neurons=100,
n_regressors=1,
n_epochs=100,
loss='gaussian_nll',
seed=seed,
)
elif regress_type == 'mlper5':
regressor = mlp_ensemble(
n_neurons=200,
n_regressors=5,
n_epochs=100,
seed=seed,
)
elif regress_type == 'mlper5g':
regressor = mlp_ensemble(
n_neurons=200,
n_regressors=5,
n_epochs=50,
loss='gaussian_nll',
seed=seed,
)
elif regress_type == 'bayesnn':
from bayesian_neural_network import BayesianNN
regressor = BayesianNN(
n_hidden1=200,
n_hidden2=200,
n_iter=1000,
n_posterior_samples=100,
random_state=seed,
verbose=True,
)
elif regress_type == 'cmf':
from cmf_regressor import CMFRegressor
regressor = CMFRegressor(
n_components=30,
seed=seed,
)
regressor.fit(
kwargs['chems'],
kwargs['prots'],
kwargs['chem2feature'],
kwargs['prot2feature'],
kwargs['Kds'],
kwargs['idx_obs'],
)
elif regress_type == 'gp':
from gaussian_process import GPRegressor
from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C
regressor = GPRegressor(
kernel=C(10000., 'fixed') * RBF(1., 'fixed'),
backend='sklearn',
n_jobs=10,
verbose=True
)
elif regress_type == 'gpfactorized':
from gaussian_process import GPRegressor
from sklearn.gaussian_process.kernels import ConstantKernel as C
from kernels import FactorizedRBF
n_features_chem = kwargs['n_features_chem']
n_features_prot = kwargs['n_features_prot']
regressor = GPRegressor(
kernel=C(1., 'fixed') * FactorizedRBF(
[ 1.1, 1. ], [ n_features_chem, n_features_prot ], 'fixed'
),
backend='sklearn',
n_restarts=0,
n_jobs=10,
verbose=True
)
elif regress_type == 'sparsegp':
from gaussian_process import SparseGPRegressor
regressor = SparseGPRegressor(
method='geosketch',
n_inducing=8000,
backend='sklearn',
n_restarts=10,
n_jobs=10,
verbose=True
)
elif regress_type == 'hybrid':
from gaussian_process import GPRegressor
from hybrid import HybridMLPEnsembleGP
regressor = HybridMLPEnsembleGP(
mlp_ensemble(
n_neurons=200,
n_regressors=1,
n_epochs=50,
seed=seed,
),
GPRegressor(
backend='sklearn',#'gpytorch',
n_restarts=10,
n_jobs=10,
verbose=True,
),
)
elif regress_type == 'dhybrid':
from gaussian_process import GPRegressor
from hybrid import HybridMLPEnsembleGP
regressor = HybridMLPEnsembleGP(
mlp_ensemble(
n_neurons=1000,
n_regressors=1,
n_hidden_layers=15,
n_epochs=300,
seed=seed,
),
GPRegressor(
backend='sklearn',#'gpytorch',
n_restarts=10,
n_jobs=10,
verbose=True,
),
)
elif regress_type == 'sparsehybrid':
from gaussian_process import SparseGPRegressor
from hybrid import HybridMLPEnsembleGP
from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C
regressor = HybridMLPEnsembleGP(
mlp_ensemble(
n_neurons=200,
n_regressors=1,
n_epochs=50,
seed=seed,
),
SparseGPRegressor(
method='geosketch',
n_inducing=8000,
backend='sklearn',
n_restarts=10,
n_jobs=10,
verbose=True
),
)
if regress_type not in { 'cmf' }:
regressor.fit(X_obs, y_obs)
#print(regressor.model_.kernel_.get_params()) # Debug.
kwargs['regressor'] = regressor
return kwargs
def learn_success_model(self, action_parameters: np.ndarray,
labels: np.ndarray) -> None:
kernel = C(1., (0.1, 100)) * RBF(1., (1e-2, 1e2))
gpr = GaussianProcessRegressor(kernel=kernel).fit(
action_parameters, labels)
self.gpr = gpr
import numpy as np
import pandas as pd
from sklearn.model_selection import ShuffleSplit, train_test_split
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import WhiteKernel as W, ConstantKernel as C, Matern, RBF, ExpSineSquared, RationalQuadratic
from sklearn.metrics import r2_score
np.random.seed(0)
data = pd.read_csv("/home/lguo1/Desktop/train.csv")
# print('data\n%s'%(data.head()))
prices = data['medv']
features = data[['rm', 'lstat', 'ptratio', 'black']]
X_train, X_test, y_train, y_test = train_test_split(features,
prices,
test_size=1 / 3)
#kernel = C(10, (1e-3, 1e3)) + C(1, (1e-3, 1e3)) * Matern(length_scale=1.0, length_scale_bounds=(1e-2, 1e2), nu=1.5) + W()
#kernel = C(10, (1e-3, 1e3)) + C(1, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2)) + W()
#kernel = C(10, (1e-3, 1e3)) + C(1, (1e-3, 1e3)) * ExpSineSquared(length_scale=1.0, periodicity=3.0, length_scale_bounds=(0.1, 10.0), periodicity_bounds=(1.0, 10.0)) + W()
#kernel = C(10, (1e-3, 1e3)) + C(1, (1e-3, 1e3)) * RationalQuadratic(length_scale=1.0, alpha=0.1) + W()
kernel = W() + C(10, (1e-3, 1e3)) + C(1, (1e-3, 1e3)) * RBF(
10, (1e-2, 1e2)) + C(1, (1e-3, 1e3)) * Matern(
length_scale=1.0, length_scale_bounds=(1e-2, 1e2), nu=1.5)
#kernel = W() + C(10, (1e-3, 1e3)) + RBF(10, (1e-3, 1e3)) + Matern(length_scale=1.0, length_scale_bounds=(1e-3, 1e3), nu=1.5)
gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9)
gp.fit(X_train, y_train)
print("Posterior (kernel: %s)\n Log-Likelihood: %.3f" %
(gp.kernel_, gp.log_marginal_likelihood(gp.kernel_.theta)))
print('training result: %f' % r2_score(y_train, gp.predict(X_train)))
print('test result: %f' % r2_score(y_test, gp.predict(X_test)))
import pytest
def f(x):
return x * np.sin(x)
X = np.atleast_2d([1., 3., 5., 6., 7., 8.]).T
X2 = np.atleast_2d([2., 4., 5.5, 6.5, 7.5]).T
y = f(X).ravel()
fixed_kernel = RBF(length_scale=1.0, length_scale_bounds="fixed")
kernels = [RBF(length_scale=1.0),
fixed_kernel,
RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
C(1.0, (1e-2, 1e2)) *
RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
C(1.0, (1e-2, 1e2)) *
RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)) +
C(1e-5, (1e-5, 1e2)),
C(0.1, (1e-2, 1e2)) *
RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)) +
WhiteKernel(1e-2, (1e-5, 1e2))
]
non_fixed_kernels = [kernel for kernel in kernels
if kernel != fixed_kernel]
@pytest.mark.parametrize('kernel', kernels)
def test_gpr_interpolation(kernel):
# Test the interpolating property for different kernels.
assert_almost_equal, assert_equal, assert_raise_message)
def f(x):
return x * np.sin(x)
X = np.atleast_2d([1., 3., 5., 6., 7., 8.]).T
X2 = np.atleast_2d([2., 4., 5.5, 6.5, 7.5]).T
y = f(X).ravel()
fixed_kernel = RBF(length_scale=1.0, length_scale_bounds="fixed")
kernels = [
RBF(length_scale=1.0), fixed_kernel,
RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
C(1.0,
(1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
C(1.0,
(1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)) +
C(1e-5, (1e-5, 1e2)),
C(0.1,
(1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)) +
C(1e-5, (1e-5, 1e2))
]
def test_gpr_interpolation():
# Test the interpolating property for different kernels.
for kernel in kernels:
gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
y_pred, y_cov = gpr.predict(X, return_cov=True)
from sklearn.utils.testing \
import (assert_true, assert_greater, assert_array_less,
assert_almost_equal, assert_equal)
def f(x):
return x * np.sin(x)
X = np.atleast_2d([1., 3., 5., 6., 7., 8.]).T
X2 = np.atleast_2d([2., 4., 5.5, 6.5, 7.5]).T
y = f(X).ravel()
fixed_kernel = RBF(length_scale=1.0, length_scale_bounds="fixed")
kernels = [RBF(length_scale=1.0), fixed_kernel,
RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
C(1.0, (1e-2, 1e2))
* RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
C(1.0, (1e-2, 1e2))
* RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3))
+ C(1e-5, (1e-5, 1e2)),
C(0.1, (1e-2, 1e2))
* RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3))
+ C(1e-5, (1e-5, 1e2))]
def test_gpr_interpolation():
"""Test the interpolating property for different kernels."""
for kernel in kernels:
gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y)
y_pred, y_cov = gpr.predict(X, return_cov=True)
def regress_variable_on_pseudotime(pseudotime,
vals,
TrajName,
var_name,
var_type,
producePlot=True,
verbose=False,
Continuous_Regression_Type='linear',
R2_Threshold=0.5,
max_sample=-1,
alpha_factor=2):
"""
Auxillary function for regression_of_variable_with_trajectories()
Continuous_Regression_Type can be 'linear','gpr' for Gaussian Process, 'kr' for kernel ridge
"""
if var_type == 'BINARY':
#convert back to binary vals
mn = min(vals)
mx = max(vals)
vals[np.where(vals == mn)] = 0
vals[np.where(vals == mx)] = 1
if len(np.unique(vals)) == 1:
regressor = None
else:
regressor = LogisticRegression(random_state=0,
max_iter=1000,
penalty='none').fit(
pseudotime, vals)
if var_type == 'CATEGORICAL':
if len(np.unique(vals)) == 1:
regressor = None
else:
regressor = LogisticRegression(random_state=0,
max_iter=1000,
penalty='none').fit(
pseudotime, vals)
if var_type == 'CONTINUOUS' or var_type == 'ORDINAL':
if len(np.unique(vals)) == 1:
regressor = None
else:
if Continuous_Regression_Type == 'gpr':
# subsampling if needed
pst = pseudotime.copy()
vls = vals.copy()
if max_sample > 0:
l = list(range(len(vals)))
random.shuffle(l)
index_value = random.sample(l, min(max_sample, len(vls)))
pst = pst[index_value]
vls = vls[index_value]
if len(np.unique(vls)) > 1:
gp_kernel = C(1.0, (1e-3, 1e3)) * RBF(1, (1e-2, 1e2))
#gp_kernel = RBF(np.std(vals))
regressor = GaussianProcessRegressor(kernel=gp_kernel,
alpha=np.var(vls) *
alpha_factor)
regressor.fit(pst, vls)
else:
regressor = None
if Continuous_Regression_Type == 'linear':
regressor = LinearRegression()
regressor.fit(pseudotime, vals)
r2score = 0
if regressor is not None:
r2score = r2_score(vals, regressor.predict(pseudotime))
if producePlot and r2score > R2_Threshold:
plt.plot(pseudotime, vals, 'ro', label='data')
unif_pst = np.linspace(min(pseudotime), max(pseudotime), 100)
pred = regressor.predict(unif_pst)
if var_type == 'BINARY' or var_type == 'CATEGORICAL':
prob = regressor.predict_proba(unif_pst)
plt.plot(unif_pst,
prob[:, 1],
'g-',
linewidth=2,
label='proba')
if var_type == 'CONTINUOUS' or var_type == 'ORDINAL':
plt.plot(unif_pst, pred, 'g-', linewidth=2, label='predicted')
bincenters, wav = moving_weighted_average(pseudotime,
vals.reshape(-1, 1),
step_size=1.5)
plt.plot(bincenters,
fill_gaps_in_number_sequence(wav),
'b-',
linewidth=2,
label='sliding av')
plt.xlabel('Pseudotime', fontsize=20)
plt.ylabel(var_name, fontsize=20)
plt.title(TrajName + ', r2={:2.2f}'.format(r2score), fontsize=20)
plt.legend(fontsize=15)
plt.show()
return r2score, regressor
def predict(self, X, return_std=False, return_cov=False):
"""Predict using the Gaussian process regression model
We can also predict based on an unfitted model by using the GP prior.
In addition to the mean of the predictive distribution, also its
standard deviation (return_std=True) or covariance (return_cov=True).
Note that at most one of the two can be requested.
Parameters
----------
X : array-like, shape = (n_samples, n_features)
Query points where the GP is evaluated
return_std : bool, default: False
If True, the standard-deviation of the predictive distribution at
the query points is returned along with the mean.
return_cov : bool, default: False
If True, the covariance of the joint predictive distribution at
the query points is returned along with the mean
Returns
-------
y_mean : array, shape = (n_samples, [n_output_dims])
Mean of predictive distribution a query points
y_std : array, shape = (n_samples,), optional
Standard deviation of predictive distribution at query points.
Only returned when return_std is True.
y_cov : array, shape = (n_samples, n_samples), optional
Covariance of joint predictive distribution a query points.
Only returned when return_cov is True.
"""
if return_std and return_cov:
raise RuntimeError(
"Not returning standard deviation of predictions when "
"returning full covariance.")
X = check_array(X)
if not hasattr(self, "X_train_"): # Unfitted;predict based on GP prior
if self.kernel is None:
kernel = (C(1.0, constant_value_bounds="fixed") *
RBF(1.0, length_scale_bounds="fixed"))
else:
kernel = self.kernel
y_mean = np.zeros(X.shape[0])
if return_cov:
y_cov = kernel(X)
return y_mean, y_cov
elif return_std:
y_var = kernel.diag(X)
return y_mean, np.sqrt(y_var)
else:
return y_mean
else: # Predict based on GP posterior
K_trans = self.kernel_(X, self.X_train_)
y_mean = K_trans.dot(self.alpha_) # Line 4 (y_mean = f_star)
y_mean = self._y_train_mean + y_mean # undo normal.
if return_cov:
v = cho_solve((self.L_, True), K_trans.T) # Line 5
y_cov = self.kernel_(X) - K_trans.dot(v) # Line 6
return y_mean, y_cov
elif return_std:
# cache result of K_inv computation
if self._K_inv is None:
# compute inverse K_inv of K based on its Cholesky
# decomposition L and its inverse L_inv
L_inv = solve_triangular(self.L_.T,
np.eye(self.L_.shape[0]))
self._K_inv = L_inv.dot(L_inv.T)
# Compute variance of predictive distribution
y_var = self.kernel_.diag(X)
y_var -= np.einsum("ij,ij->i", np.dot(K_trans, self._K_inv),
K_trans)
# Check if any of the variances is negative because of
# numerical issues. If yes: set the variance to 0.
y_var_negative = y_var < 0
if np.any(y_var_negative):
warnings.warn("Predicted variances smaller than 0. "
"Setting those variances to 0.")
y_var[y_var_negative] = 0.0
return y_mean, np.sqrt(y_var)
else:
return y_mean
bkghist_template.Rebin(8)
bkghist_template.Scale(stats/bkghist_template.Integral())
trainHisto = bkghist_template.Clone('trainHisto')
optKernel = None
GPh = GPHisto(bkghist_template)
X = GPh.getXArr()
y = GPh.getYArr()
dy = GPh.getErrArr()
x = GPh.getXArr()
kernel = C(10.0, (1e-3, 1e15)) * RBF(50.0, (1e-2, 1e5)) #squared exponential kernel
#kernel = C(1000.0, (1e-3, 1e15)) * FallExp(1.0, (1e-5, 1e2), 1.0, (1e-3,1e15)) * Gibbs(0.01, (1e-7, 1e5), 0.01, (1e-7,1e5))
gp = GaussianProcessRegressor(kernel=kernel
#,optimizer='fmin'
,alpha=dy**2
,n_restarts_optimizer=15
)
gp.fit(X,y)
print gp.kernel_
lengthOpt = float(re.search('length_scale=(\d+(\.\d+)?)', gp.kernel_.__repr__()).group(1))
#raise()
optKernel = gp.kernel_
X = np.atleast_2d(np.linspace(0, 10, 30)).T
X2 = np.atleast_2d([2., 4., 5.5, 6.5, 7.5]).T
y = np.array(f(X).ravel() > 0, dtype=int)
fX = f(X).ravel()
y_mc = np.empty(y.shape, dtype=int) # multi-class
y_mc[fX < -0.35] = 0
y_mc[(fX >= -0.35) & (fX < 0.35)] = 1
y_mc[fX > 0.35] = 2
fixed_kernel = RBF(length_scale=1.0, length_scale_bounds="fixed")
kernels = [
RBF(length_scale=0.1), fixed_kernel,
RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)),
C(1.0,
(1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3))
]
non_fixed_kernels = [kernel for kernel in kernels if kernel != fixed_kernel]
@pytest.mark.parametrize('kernel', kernels)
def test_predict_consistent(kernel):
# Check binary predict decision has also predicted probability above 0.5.
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
assert_array_equal(gpc.predict(X), gpc.predict_proba(X)[:, 1] >= 0.5)
def test_predict_consistent_structured():
# Check binary predict decision has also predicted probability above 0.5.
X = ['A', 'AB', 'B']
y = np.array([True, False, True])
#######Special Credits for developing the code : AV Sir's lecture material ###########
import pandas as pd
import numpy as np
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C
from matplotlib import pyplot as plt
data = [[0, -45], [2, -58], [4, -36], [6, -59], [8, -36], [10, -55], [11, -64]]
Train = pd.DataFrame(data, columns=['Distance', 'Strength'])
data1 = [[1, -51], [3, -63], [5, -52], [7, -62], [9, -43]]
Test = pd.DataFrame(data1, columns=['Distance', 'Strength'])
kernel = C(1.0, (1e-10, 1e10)) * RBF(2, (1e-4, 1e4))
gp = GaussianProcessRegressor(kernel=kernel,
n_restarts_optimizer=10,
random_state=2)
#print(gp)
gp.fit((np.array(Train["Distance"])).reshape(-1, 1), Train["Strength"])
y_pred, sigma = gp.predict((np.array(Test["Distance"])).reshape(-1, 1),
return_std=True)
for i in range(0, len(y_pred)):
print("Distance", Test["Distance"][i])
print("Actual value of Signal Strength", Test["Strength"][i])
print("predicted mean of Signal Strength", y_pred[i])
print("variance", sigma[i])
print("\n")
#print(Test["Strength"])
