def kernel_Matern(k=1e0, l_t=5e3, l_l=5e3): kernel = C(k, (1e-2, 1e4)) * Matern( (l_t, l_l), ((1e0, 1e6), (1e0, 1e6)), nu=1.5) return kernel
def fit(self, X, y): """Fit Gaussian process regression model. Parameters ---------- X : array-like, shape = (n_samples, n_features) Training data y : array-like, shape = (n_samples, [n_output_dims]) Target values Returns ------- self : returns an instance of self. """ if self.kernel is None: # Use an RBF kernel as default self.kernel_ = C(1.0, constant_value_bounds="fixed") \ * RBF(1.0, length_scale_bounds="fixed") else: self.kernel_ = clone(self.kernel) self._rng = check_random_state(self.random_state) X, y = check_X_y(X, y, multi_output=True, y_numeric=True) # Normalize target value if self.normalize_y: self._y_train_mean = np.mean(y, axis=0) # demean y y = y - self._y_train_mean else: self._y_train_mean = np.zeros(1) if np.iterable(self.alpha) \ and self.alpha.shape[0] != y.shape[0]: if self.alpha.shape[0] == 1: self.alpha = self.alpha[0] else: raise ValueError( "alpha must be a scalar or an array" " with same number of entries as y.(%d != %d)" % (self.alpha.shape[0], y.shape[0])) self.X_train_ = np.copy(X) if self.copy_X_train else X self.y_train_ = np.copy(y) if self.copy_X_train else y if self.optimizer is not None and self.kernel_.n_dims > 0: # Choose hyperparameters based on maximizing the log-marginal # likelihood (potentially starting from several initial values) def obj_func(theta, eval_gradient=True): if eval_gradient: lml, grad = self.log_marginal_likelihood( theta, eval_gradient=True) return -lml, -grad else: return -self.log_marginal_likelihood(theta) # First optimize starting from theta specified in kernel optima = [(self._constrained_optimization(obj_func, self.kernel_.theta, self.kernel_.bounds))] # Additional runs are performed from log-uniform chosen initial # theta if self.n_restarts_optimizer > 0: if not np.isfinite(self.kernel_.bounds).all(): raise ValueError( "Multiple optimizer restarts (n_restarts_optimizer>0) " "requires that all bounds are finite.") bounds = self.kernel_.bounds for iteration in range(self.n_restarts_optimizer): theta_initial = \ self._rng.uniform(bounds[:, 0], bounds[:, 1]) optima.append( self._constrained_optimization(obj_func, theta_initial, bounds)) # Select result from run with minimal (negative) log-marginal # likelihood lml_values = list(map(itemgetter(1), optima)) self.kernel_.theta = optima[np.argmin(lml_values)][0] self.log_marginal_likelihood_value_ = -np.min(lml_values) else: self.log_marginal_likelihood_value_ = \ self.log_marginal_likelihood(self.kernel_.theta) # Precompute quantities required for predictions which are independent # of actual query points K = self.kernel_(self.X_train_) K[np.diag_indices_from(K)] += self.alpha try: self.L_ = cholesky(K, lower=True) # Line 2 # self.L_ changed, self._K_inv needs to be recomputed self._K_inv = None except np.linalg.LinAlgError as exc: exc.args = ("The kernel, %s, is not returning a " "positive definite matrix. Try gradually " "increasing the 'alpha' parameter of your " "GaussianProcessRegressor estimator." % self.kernel_, ) + exc.args raise self.alpha_ = cho_solve((self.L_, True), self.y_train_) # Line 3 return self
return x * np.sin(x) # ---------------------------------------------------------------------- # First the noiseless case X = np.atleast_2d([1., 3., 5., 6., 7., 8.]).T # Observations y = f(X).ravel() # Mesh the input space for evaluations of the real function, the prediction and # its MSE x = np.atleast_2d(np.linspace(0, 10, 1000)).T # Instantiate a Gaussian Process model kernel = C(1.0, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2)) gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9) # Fit to data using Maximum Likelihood Estimation of the parameters gp.fit(X, y) # Make the prediction on the meshed x-axis (ask for MSE as well) y_pred, sigma = gp.predict(x, return_std=True) # Plot the function, the prediction and the 95% confidence interval based on # the MSE plt.figure() plt.plot(x, f(x), 'r:', label=u'$f(x) = x\,\sin(x)$') plt.plot(X, y, 'r.', markersize=10, label=u'Observations') plt.plot(x, y_pred, 'b-', label=u'Prediction') plt.fill(np.concatenate([x, x[::-1]]),
#!/usr/bin/env python #-*-coding:utf-8-*- #user:xieyuhui import numpy as np from sklearn.gaussian_process.kernels import RBF, Matern, RationalQuadratic, ExpSineSquared, DotProduct, ConstantKernel as C import pylab from sklearn.gaussian_process import GaussianProcessRegressor #X_ = np.array([[0], [1.1], [1.3], [2.2], [2.8], [3.6], [3.7], [4.6], [4.7], [4.8]]) #Y_ = np.array([[-0.1], [0.9], [1], [0.1], [0], [0.7], [0.8], [-1.1], [-1], [-0.8]]) X = np.linspace(0, 5, 100)[:, None] # RBF kernel = C(1.0) * RBF(length_scale=1) gp = GaussianProcessRegressor(kernel=kernel, alpha=1e-5, n_restarts_optimizer=10) pylab.figure(0, figsize=(14, 12)) pylab.subplot(3, 2, 1) ymean, y_std = gp.predict(X, return_std=True) pylab.plot(X, ymean, 'k', lw=3, zorder=9, label="mean") pylab.fill_between(X[:, 0], ymean - y_std, ymean + y_std, alpha=0.5, color='k') y_samples = gp.sample_y(X, 10) pylab.plot(X, y_samples, color='b', lw=2) pylab.plot(X, y_samples[:, 0], color='b', lw=2, label="sample") pylab.legend(loc="best") pylab.xlim(0, 5) pylab.ylim(-3, 3) pylab.title("Prior Samples")
# reload = True reload = False n_iter = 1000 N_EARLY_STOPPING = 1000 # ALPHA = MEAN # prior: ALPHA = 0.001 # ndim ** 1 GAMMA = 10**(-2) * 2 * ndim GAMMA0 = 0.01 * GAMMA GAMMA_Y = 10**(-2) # weight of adjacen IS_EDGE_NORMALIZED = True # kernel = Matern(nu=2.5) kernel = C(1) * RBF(2) # kernel = None BURNIN = False # TODO INITIAL_K = 10 INITIAL_THETA = 10 UPDATE_HYPERPARAM_FUNC = 'pairwise_sampling' # None output_dir = 'output' # _gmrf_min0max1_easy' parameter_dir = os.path.join('param_dir', 'csv_files') result_filename = os.path.join(output_dir, 'gaussian_result_2dim.csv') ACQUISITION_FUNC = 'ts' # 'ei' ACQUISITION_PARAM_DIC = { 'beta': 5, #for "ucb"
def parameterized_inference(algorithm='carl', morphing_aware=False, training_sample='baseline', use_smearing=False, denominator=0, alpha=None, training_sample_size=None, do_neyman=False, options=''): """ Trains and evaluates one of the parameterized higgs_inference methods. Input: training_sample_size: denominator: algorithm: Type of the algorithm used. Currently supported: 'carl', 'score', 'combined', 'regression', and 'combinedregression'. morphing_aware: bool that decides whether a morphing-aware or morphing-agnostic architecture is used. training_sample: Training sample. Can be 'baseline', 'basis', or 'random'. use_smearing: alpha: Factor that weights the score term in the if algorithm is 'combined' or 'combinedregression'. do_neyman: Switches on the evaluation of toy experiments for the Neyman construction. options: Further options in a list of strings or string. Output: """ logging.info('----> Starting parameterized inference <----') assert algorithm in ['carl', 'score', 'combined', 'regression', 'combinedregression'] assert training_sample in ['baseline', 'basis', 'random'] ################################################################################ # Settings ################################################################################ random_theta_mode = training_sample == 'random' basis_theta_mode = training_sample == 'basis' learn_logr_mode = ('learns' not in options) new_sample_mode = ('new' in options) short_mode = ('short' in options) long_mode = ('long' in options) deep_mode = ('deep' in options) shallow_mode = ('shallow' in options) debug_mode = ('debug' in options) factor_out_sm_in_aware_mode = morphing_aware and ('factorsm' in options) small_lr_mode = ('slowlearning' in options) large_lr_mode = ('fastlearning' in options) large_batch_mode = ('largebatch' in options) small_batch_mode = ('smallbatch' in options) constant_lr_mode = ('constantlr' in options) neyman2_mode = ('neyman2' in options) neyman3_mode = ('neyman3' in options) recalibration_mode = ('recalibration' in options) filename_addition = '' if morphing_aware: filename_addition = '_aware' if random_theta_mode: filename_addition += '_random' elif basis_theta_mode: filename_addition += '_basis' if not learn_logr_mode: filename_addition += '_learns' if factor_out_sm_in_aware_mode: filename_addition += '_factorsm' learning_rate = settings.learning_rate_default if small_lr_mode: filename_addition += '_slowlearning' learning_rate = settings.learning_rate_small elif large_lr_mode: filename_addition += '_fastlearning' learning_rate = settings.learning_rate_large lr_decay = settings.learning_rate_decay if constant_lr_mode: lr_decay = 0. filename_addition += '_constantlr' batch_size = settings.batch_size_default if large_batch_mode: filename_addition += '_largebatch' batch_size = settings.batch_size_large elif small_batch_mode: filename_addition += '_smallbatch' batch_size = settings.batch_size_small settings.batch_size = batch_size alpha_regression = settings.alpha_regression_default alpha_carl = settings.alpha_carl_default if alpha is not None: alpha_regression = alpha alpha_carl = alpha precision = int(max(- math.floor(np.log10(alpha)) + 1, 0)) filename_addition += '_alpha_' + format_number(alpha, precision) n_hidden_layers = settings.n_hidden_layers_default if shallow_mode: n_hidden_layers = settings.n_hidden_layers_shallow filename_addition += '_shallow' elif deep_mode: n_hidden_layers = settings.n_hidden_layers_deep filename_addition += '_deep' n_epochs = settings.n_epochs_default early_stopping = True early_stopping_patience = settings.early_stopping_patience if debug_mode: n_epochs = settings.n_epochs_short early_stopping = False filename_addition += '_debug' elif long_mode: n_epochs = settings.n_epochs_long filename_addition += '_long' elif short_mode: n_epochs = settings.n_epochs_short early_stopping = False filename_addition += '_short' if training_sample_size is not None: filename_addition += '_trainingsamplesize_' + str(training_sample_size) n_epoch_factor = int(len(settings.thetas_train) * \ (settings.n_events_baseline_num + \ settings.n_events_baseline_den)/training_sample_size) n_epochs *= n_epoch_factor lr_decay /= float(n_epoch_factor) early_stopping_patience *= n_epoch_factor input_X_prefix = '' if use_smearing: input_X_prefix = 'smeared_' filename_addition += '_smeared' theta1 = settings.theta1_default input_filename_addition = '' if denominator > 0: input_filename_addition = '_denom' + str(denominator) filename_addition += '_denom' + str(denominator) theta1 = settings.theta1_alternatives[denominator - 1] if new_sample_mode: filename_addition += '_new' input_filename_addition += '_new' n_expected_events_neyman = settings.n_expected_events_neyman n_neyman_null_experiments = settings.n_neyman_null_experiments n_neyman_alternate_experiments = settings.n_neyman_alternate_experiments neyman_filename = 'neyman' if neyman2_mode: neyman_filename = 'neyman2' n_expected_events_neyman = settings.n_expected_events_neyman2 n_neyman_null_experiments = settings.n_neyman2_null_experiments n_neyman_alternate_experiments = settings.n_neyman2_alternate_experiments if neyman3_mode: neyman_filename = 'neyman3' n_expected_events_neyman = settings.n_expected_events_neyman3 n_neyman_null_experiments = settings.n_neyman3_null_experiments n_neyman_alternate_experiments = settings.n_neyman3_alternate_experiments results_dir = settings.base_dir + '/results/parameterized' neyman_dir = settings.neyman_dir + '/parameterized' logging.info('Main settings:') logging.info(' Algorithm: %s', algorithm) logging.info(' Morphing-aware: %s', morphing_aware) logging.info(' Training sample: %s', training_sample) logging.info(' Denominator theta: denominator %s = theta %s = %s', denominator, theta1, settings.thetas[theta1]) logging.info('Options:') logging.info(' Number of hidden layers: %s', n_hidden_layers) if algorithm == 'combined': logging.info(' alpha: %s', alpha_carl) elif algorithm == 'combinedregression': logging.info(' alpha: %s', alpha_regression) logging.info(' Batch size: %s', batch_size) logging.info(' Learning rate: %s', learning_rate) logging.info(' Learning rate decay: %s', lr_decay) logging.info(' Number of epochs: %s', n_epochs) logging.info(' Training samples: %s', 'all' if training_sample_size is None else training_sample_size) if do_neyman: logging.info(' NC experiments: (%s alternate + %s null) ' + \ 'experiments with %s alternate events each', n_neyman_alternate_experiments, n_neyman_null_experiments, n_expected_events_neyman) else: logging.info(' NC experiments: False') logging.info(' Debug mode: %s', debug_mode) ################################################################################ # Data ################################################################################ # Load data train_filename = '_train' if random_theta_mode: train_filename += '_random' elif basis_theta_mode: train_filename += '_basis' train_filename += input_filename_addition X_train = np.load(settings.unweighted_events_dir + '/' + \ input_X_prefix + 'X' + train_filename + '.npy') X_train_unshuffled = np.load(settings.unweighted_events_dir + '/' + \ input_X_prefix + 'X' + train_filename + '.npy') y_train = np.load(settings.unweighted_events_dir + '/y' + train_filename + '.npy') scores_train = np.load(settings.unweighted_events_dir + '/scores' + \ train_filename + '.npy') r_train = np.load(settings.unweighted_events_dir + '/r' + train_filename + '.npy') theta0_train = np.load(settings.unweighted_events_dir + '/theta0' + \ train_filename + '.npy') theta0_train_unshuffled = np.load(settings.unweighted_events_dir + \ '/theta0' + train_filename + '.npy') X_calibration = np.load(settings.unweighted_events_dir + '/' + \ input_X_prefix + 'X_calibration' + \ input_filename_addition + '.npy') weights_calibration = np.load(settings.unweighted_events_dir + \ '/weights_calibration' + \ input_filename_addition + '.npy') if recalibration_mode: X_recalibration = np.load( settings.unweighted_events_dir + '/' + input_X_prefix + 'X_recalibration' + '.npy') X_test = np.load( settings.unweighted_events_dir + '/' + input_X_prefix + 'X_test' + input_filename_addition + '.npy') r_test = np.load(settings.unweighted_events_dir + '/r_test' + input_filename_addition + '.npy') X_roam = np.load( settings.unweighted_events_dir + '/' + input_X_prefix + 'X_roam' + input_filename_addition + '.npy') n_roaming = len(X_roam) X_illustration = np.load( settings.unweighted_events_dir + '/' + input_X_prefix + 'X_illustration' + input_filename_addition + '.npy') if do_neyman: X_neyman_alternate = np.load( settings.unweighted_events_dir + '/neyman/' + input_X_prefix + 'X_' + neyman_filename + '_alternate.npy') n_events_test = X_test.shape[0] assert settings.n_thetas == r_test.shape[0] # Shuffle training data X_train, y_train, scores_train, r_train, theta0_train = shuffle(X_train, y_train, scores_train, r_train, theta0_train, random_state=44) # Limit training sample size keras_verbosity = 2 if training_sample_size is not None: keras_verbosity = 0 original_training_sample_size = X_train.shape[0] X_train = X_train[:training_sample_size] y_train = y_train[:training_sample_size] scores_train = scores_train[:training_sample_size] r_train = r_train[:training_sample_size] theta0_train = theta0_train[:training_sample_size] logging.info('Reduced training sample size from %s to %s (factor %s)', original_training_sample_size, X_train.shape[0], n_epoch_factor) # Normalize data scaler = StandardScaler() scaler.fit(np.array(X_train, dtype=np.float64)) X_train_transformed = scaler.transform(X_train) X_train_transformed_unshuffled = scaler.transform(X_train_unshuffled) X_test_transformed = scaler.transform(X_test) X_roam_transformed = scaler.transform(X_roam) X_calibration_transformed = scaler.transform(X_calibration) X_illustration_transformed = scaler.transform(X_illustration) if recalibration_mode: X_recalibration_transformed = scaler.transform(X_recalibration) if do_neyman: X_neyman_alternate_transformed = scaler.transform( X_neyman_alternate.reshape((-1, X_neyman_alternate.shape[2]))) # Roaming data X_thetas_train = np.hstack((X_train_transformed, theta0_train)) X_thetas_train_unshuffled = np.hstack((X_train_transformed_unshuffled, theta0_train_unshuffled)) y_logr_score_train = np.hstack((y_train.reshape(-1, 1), np.log(r_train).reshape((-1, 1)), scores_train)) xi = np.linspace(-1.0, 1.0, settings.n_thetas_roam) yi = np.linspace(-1.0, 1.0, settings.n_thetas_roam) xx, yy = np.meshgrid(xi, yi) thetas_roam = np.asarray((xx.flatten(), yy.flatten())).T X_thetas_roam = [] for i in range(n_roaming): X_thetas_roam.append(np.zeros((settings.n_thetas_roam ** 2, X_roam_transformed.shape[1] + 2))) X_thetas_roam[-1][:, :-2] = X_roam_transformed[i, :] X_thetas_roam[-1][:, -2:] = thetas_roam if debug_mode: X_thetas_train = X_thetas_train[::100] y_logr_score_train = y_logr_score_train[::100] X_test_transformed = X_test[::100] r_test = r_test[:, ::100] X_calibration_transformed = X_calibration_transformed[::100] weights_calibration = weights_calibration[:, ::100] X_illustration_transformed = X_illustration_transformed[::100] if recalibration_mode: X_recalibration_transformed = X_recalibration_transformed[::100] n_events_test = len(X_test_transformed) ################################################################################ # Training ################################################################################ if algorithm == 'carl': if morphing_aware: regr = KerasRegressor(lambda: make_classifier_carl_morphingaware(n_hidden_layers=n_hidden_layers, learn_log_r=learn_logr_mode, learning_rate=learning_rate), epochs=n_epochs, validation_split=settings.validation_split, verbose=keras_verbosity) else: # Agnostic Parametrization ??? regr = KerasRegressor(lambda: make_classifier_carl(n_hidden_layers=n_hidden_layers, learn_log_r=learn_logr_mode, learning_rate=learning_rate), epochs=n_epochs, validation_split=settings.validation_split, verbose=keras_verbosity) elif algorithm == 'score': if morphing_aware: regr = KerasRegressor(lambda: make_classifier_score_morphingaware(n_hidden_layers=n_hidden_layers, learn_log_r=learn_logr_mode, learning_rate=learning_rate), epochs=n_epochs, validation_split=settings.validation_split, verbose=keras_verbosity) else: # Agnostic Parametrization ??? regr = KerasRegressor(lambda: make_classifier_score(n_hidden_layers=n_hidden_layers, learn_log_r=learn_logr_mode, learning_rate=learning_rate), epochs=n_epochs, validation_split=settings.validation_split, verbose=keras_verbosity) elif algorithm == 'combined': if morphing_aware: regr = KerasRegressor(lambda: make_classifier_combined_morphingaware(n_hidden_layers=n_hidden_layers, learn_log_r=learn_logr_mode, alpha=alpha_carl, learning_rate=learning_rate), epochs=n_epochs, validation_split=settings.validation_split, verbose=keras_verbosity) else: regr = KerasRegressor(lambda: make_classifier_combined(n_hidden_layers=n_hidden_layers, learn_log_r=learn_logr_mode, alpha=alpha_carl, learning_rate=learning_rate), epochs=n_epochs, validation_split=settings.validation_split, verbose=keras_verbosity) elif algorithm == 'regression': if morphing_aware: regr = KerasRegressor(lambda: make_regressor_morphingaware(n_hidden_layers=n_hidden_layers, factor_out_sm=factor_out_sm_in_aware_mode, learning_rate=learning_rate), epochs=n_epochs, validation_split=settings.validation_split, verbose=keras_verbosity) else: regr = KerasRegressor(lambda: make_regressor(n_hidden_layers=n_hidden_layers), epochs=n_epochs, validation_split=settings.validation_split, verbose=keras_verbosity) elif algorithm == 'combinedregression': if morphing_aware: regr = KerasRegressor(lambda: make_combined_regressor_morphingaware(n_hidden_layers=n_hidden_layers, factor_out_sm=factor_out_sm_in_aware_mode, alpha=alpha_regression, learning_rate=learning_rate), epochs=n_epochs, validation_split=settings.validation_split, verbose=keras_verbosity) else: regr = KerasRegressor(lambda: make_combined_regressor(n_hidden_layers=n_hidden_layers, alpha=alpha_regression, learning_rate=learning_rate), epochs=n_epochs, validation_split=settings.validation_split, verbose=keras_verbosity) else: raise ValueError() # Callbacks callbacks = [] detailed_history = {} callbacks.append(DetailedHistory(detailed_history)) if not constant_lr_mode: def lr_scheduler(epoch): return learning_rate * np.exp(- epoch * lr_decay) callbacks.append(LearningRateScheduler(lr_scheduler)) if early_stopping: callbacks.append(EarlyStopping(verbose=1, patience=early_stopping_patience)) # Training logging.info('Starting training') history = regr.fit(X_thetas_train[::], y_logr_score_train[::], callbacks=callbacks, batch_size=batch_size) # Save metrics def _save_metrics(key, filename): try: metrics = np.asarray([history.history[key], history.history['val_' + key]]) np.save(results_dir + '/traininghistory_' + filename + '_' + \ algorithm + filename_addition + '.npy', metrics) except KeyError: logging.warning('Key %s not found in per-epoch history. ' + \ 'Available keys: %s', key, list(history.history.keys())) try: detailed_metrics = np.asarray(detailed_history[key]) np.save(results_dir + '/traininghistory_100batches_' + \ filename + '_' + algorithm + filename_addition + \ '.npy', detailed_metrics) except KeyError: logging.warning('Key %s not found in per-batch history. ' + \ 'Available keys: %s', key, list(detailed_history.keys())) _save_metrics('loss', 'loss') _save_metrics('full_cross_entropy', 'ce') _save_metrics('full_mse_log_r', 'mse_logr') _save_metrics('full_mse_score', 'mse_scores') # Evaluate r_hat on training sample r_hat_train = np.exp(regr.predict(X_thetas_train_unshuffled)[:, 1]) # np.save(results_dir + '/r_train_' + algorithm + filename_addition + '.npy', r_hat_train) ################################################################################ # Raw evaluation loop ################################################################################ # carl wrapper # ratio = ClassifierScoreRatio(regr, prefit=True) logging.info('Starting evaluation') expected_llr = [] mse_log_r = [] trimmed_mse_log_r = [] eval_times = [] expected_r_vs_sm = [] if recalibration_mode: recalibration_expected_r = [] for t, theta in enumerate(settings.thetas): if (t + 1) % 100 == 0: logging.info('Starting theta %s / %s', t + 1, settings.n_thetas) ################################################################################ # Evaluation ################################################################################ # Prepare test data thetas0_array = np.zeros((X_test_transformed.shape[0], 2), dtype=X_test_transformed.dtype) thetas0_array[:, :] = theta X_thetas_test = np.hstack((X_test_transformed, thetas0_array)) # Evaluation time_before = time.time() prediction = regr.predict(X_thetas_test) eval_times.append(time.time() - time_before) this_log_r = prediction[:, 1] this_score = prediction[:, 2:4] if morphing_aware: this_wi = prediction[:, 4:19] this_ri = prediction[:, 19:] logging.debug('Morphing weights for theta %s (%s): %s', t, theta, this_wi[0]) # Extract numbers of interest expected_llr.append(- 2.*settings.n_expected_events/ \ n_events_test*np.sum(this_log_r)) mse_log_r.append(calculate_mean_squared_error(np.log(r_test[t]), this_log_r, 0.)) trimmed_mse_log_r.append(calculate_mean_squared_error(np.log(r_test[t]), this_log_r, 'auto')) if t == settings.theta_observed: r_sm = np.exp(this_log_r) expected_r_vs_sm.append(np.mean(np.exp(this_log_r) / r_sm)) # For benchmark thetas, save more info if t == settings.theta_benchmark_nottrained: np.save(results_dir + '/r_nottrained_' + algorithm + \ filename_addition + '.npy', np.exp(this_log_r)) np.save(results_dir + '/scores_nottrained_' + algorithm + \ filename_addition + '.npy', this_score) np.save(results_dir + '/r_vs_sm_nottrained_' + algorithm + \ filename_addition + '.npy', np.exp(this_log_r) / r_sm) if morphing_aware: np.save(results_dir + '/morphing_ri_nottrained_' + \ algorithm + filename_addition + '.npy', this_ri) np.save(results_dir + '/morphing_wi_nottrained_' + \ algorithm + filename_addition + '.npy', this_wi) elif t == settings.theta_benchmark_trained: np.save(results_dir + '/r_trained_' + algorithm + \ filename_addition + '.npy', np.exp(this_log_r)) np.save(results_dir + '/scores_trained_' + algorithm + \ filename_addition + '.npy', this_score) np.save(results_dir + '/r_vs_sm_trained_' + algorithm + \ filename_addition + '.npy', np.exp(this_log_r) / r_sm) if morphing_aware: np.save(results_dir + '/morphing_ri_trained_' + \ algorithm + filename_addition + '.npy', this_ri) np.save(results_dir + '/morphing_wi_trained_' + \ algorithm + filename_addition + '.npy', this_wi) ################################################################################ # Recalibration ################################################################################ if recalibration_mode: # Prepare data for recalibration thetas0_array = np.zeros((X_recalibration_transformed.shape[0], 2), dtype=X_recalibration_transformed.dtype) thetas0_array[:, :] = settings.thetas[t] X_thetas_recalibration = np.hstack((X_recalibration_transformed, thetas0_array)) # Evaluate recalibration data prediction = regr.predict(X_thetas_recalibration) this_r = np.exp(prediction[:, 1]) if t == settings.theta_observed: r_recalibration_sm = this_r recalibration_expected_r.append(np.mean(this_r / r_recalibration_sm)) ################################################################################ # Illustration ################################################################################ if t == settings.theta_benchmark_illustration: # Prepare data for illustration thetas0_array = np.zeros((X_illustration_transformed.shape[0], 2), dtype=X_illustration_transformed.dtype) thetas0_array[:, :] = settings.thetas[t] X_thetas_illustration = np.hstack((X_illustration_transformed, thetas0_array)) # Evaluate illustration data prediction = regr.predict(X_thetas_illustration) r_hat_illustration = np.exp(prediction[:, 1]) np.save(results_dir + '/r_illustration_' + algorithm + filename_addition + '.npy', r_hat_illustration) ################################################################################ # Neyman construction toys ################################################################################ if do_neyman: # Prepare alternate data for Neyman construction thetas0_array = np.zeros((X_neyman_alternate_transformed.shape[0], 2), dtype=X_neyman_alternate_transformed.dtype) thetas0_array[:, :] = theta X_thetas_neyman_alternate = np.hstack((X_neyman_alternate_transformed, thetas0_array)) # Neyman construction: evaluate alternate sample (raw) log_r_neyman_alternate = regr.predict(X_thetas_neyman_alternate)[:, 1] llr_neyman_alternate = -2. * np.sum(log_r_neyman_alternate.reshape((-1, n_expected_events_neyman)), axis=1) np.save(neyman_dir + '/' + neyman_filename + '_llr_alternate_' + str( t) + '_' + algorithm + filename_addition + '.npy', llr_neyman_alternate) # NC: null X_neyman_null = np.load(settings.unweighted_events_dir + \ '/neyman/' + input_X_prefix + 'X_' + \ neyman_filename + '_null_' + str(t) + '.npy') X_neyman_null_transformed = scaler.transform( X_neyman_null.reshape((-1, X_neyman_null.shape[2]))) # Prepare null data for Neyman construction thetas0_array = np.zeros((X_neyman_null_transformed.shape[0], 2), dtype=X_neyman_null_transformed.dtype) thetas0_array[:, :] = settings.thetas[t] X_thetas_neyman_null = np.hstack((X_neyman_null_transformed, thetas0_array)) # Neyman construction: evaluate null sample (raw) log_r_neyman_null = regr.predict(X_thetas_neyman_null)[:, 1] llr_neyman_null = -2.*np.sum(log_r_neyman_null.reshape( (-1, n_expected_events_neyman)), axis=1) np.save(neyman_dir + '/' + neyman_filename + '_llr_null_' + \ str(t) + '_' + algorithm + filename_addition + \ '.npy', llr_neyman_null) # NC: null evaluated at alternate if t == settings.theta_observed: for tt in range(settings.n_thetas): X_neyman_null = np.load(settings.unweighted_events_dir + \ '/neyman/' + input_X_prefix + \ 'X_' + neyman_filename + \ '_null_' + str(tt) + '.npy') X_neyman_null_transformed = scaler.transform( X_neyman_null.reshape((-1, X_neyman_null.shape[2]))) X_thetas_neyman_null = np.hstack((X_neyman_null_transformed, thetas0_array)) # Neyman construction: evaluate null sample (raw) log_r_neyman_null = regr.predict(X_thetas_neyman_null)[:, 1] llr_neyman_null = -2. * np.sum(log_r_neyman_null.reshape((-1, n_expected_events_neyman)), axis=1) np.save(neyman_dir + '/' + neyman_filename + \ '_llr_nullatalternate_' + str(tt) + '_' + \ algorithm + filename_addition + '.npy', llr_neyman_null) # Save evaluation results expected_llr = np.asarray(expected_llr) mse_log_r = np.asarray(mse_log_r) trimmed_mse_log_r = np.asarray(trimmed_mse_log_r) expected_r_vs_sm = np.asarray(expected_r_vs_sm) np.save(results_dir + '/llr_' + algorithm + filename_addition + '.npy', expected_llr) np.save(results_dir + '/mse_logr_' + algorithm + filename_addition + '.npy', mse_log_r) np.save(results_dir + '/trimmed_mse_logr_' + algorithm + filename_addition + '.npy', trimmed_mse_log_r) np.save(results_dir + '/expected_r_vs_sm_' + algorithm + filename_addition + '.npy', expected_r_vs_sm) if recalibration_mode: recalibration_expected_r = np.asarray(recalibration_expected_r) np.save(results_dir + '/recalibration_expected_r_vs_sm_' + \ algorithm + filename_addition + '.npy', recalibration_expected_r) # Evaluation times logging.info('Evaluation timing: median %s s, mean %s s', np.median(eval_times), np.mean(eval_times)) logging.info('Starting roaming') r_roam = [] for i in range(n_roaming): prediction = regr.predict(X_thetas_roam[i]) r_roam.append(np.exp(prediction[:, 1])) r_roam = np.asarray(r_roam) np.save(results_dir + '/r_roam_' + algorithm + filename_addition + '.npy', r_roam) ################################################################################ # Calibrated evaluation loop ################################################################################ logging.info('Starting calibrated evaluation and roaming') expected_llr_calibrated = [] mse_log_r_calibrated = [] trimmed_mse_log_r_calibrated = [] r_roam_temp = np.zeros((settings.n_thetas, n_roaming)) eval_times = [] expected_r_vs_sm = [] if recalibration_mode: recalibration_expected_r = [] for t, theta in enumerate(settings.thetas): if (t + 1) % 100 == 0: logging.info('Starting theta %s / %s', t + 1, settings.n_thetas) ################################################################################ # Calibration ################################################################################ # Prepare data for calibration n_calibration_each = X_calibration_transformed.shape[0] thetas0_array = np.zeros((n_calibration_each, 2), dtype=X_calibration_transformed.dtype) thetas0_array[:, :] = settings.thetas[t] X_thetas_calibration = np.hstack((X_calibration_transformed, thetas0_array)) X_thetas_calibration = np.vstack((X_thetas_calibration, X_thetas_calibration)) y_calibration = np.zeros(2 * n_calibration_each) y_calibration[n_calibration_each:] = 1. w_calibration = np.zeros(2 * n_calibration_each) w_calibration[:n_calibration_each] = weights_calibration[t] w_calibration[n_calibration_each:] = weights_calibration[theta1] # Calibration ratio_calibrated = ClassifierScoreRatio( CalibratedClassifierScoreCV(regr, cv='prefit', method='isotonic')) ratio_calibrated.fit(X_thetas_calibration, y_calibration, sample_weight=w_calibration) ################################################################################ # Evaluation ################################################################################ # Prepare data thetas0_array = np.zeros((X_test_transformed.shape[0], 2), dtype=X_test_transformed.dtype) thetas0_array[:, :] = settings.thetas[t] X_thetas_test = np.hstack((X_test_transformed, thetas0_array)) time_before = time.time() this_r, this_other = ratio_calibrated.predict(X_thetas_test) eval_times.append(time.time() - time_before) this_score = this_other[:, 1:3] # Extract numbers of interest expected_llr_calibrated.append(- 2.*settings.n_expected_events/ \ n_events_test*np.sum(np.log(this_r))) mse_log_r_calibrated.append( calculate_mean_squared_error(np.log(r_test[t]), np.log(this_r), 0.)) trimmed_mse_log_r_calibrated.append( calculate_mean_squared_error(np.log(r_test[t]), np.log(this_r), 'auto')) if t == settings.theta_observed: r_sm = this_r expected_r_vs_sm.append(np.mean(this_r / r_sm)) # For benchmark theta, save more data if t == settings.theta_benchmark_nottrained: np.save(results_dir + '/scores_nottrained_' + algorithm + '_calibrated' + filename_addition + '.npy', this_score) np.save(results_dir + '/r_nottrained_' + algorithm + '_calibrated' + filename_addition + '.npy', this_r) np.save(results_dir + '/r_vs_sm_nottrained_' + algorithm + '_calibrated' + filename_addition + '.npy', this_r / r_sm) np.save(results_dir + '/calvalues_nottrained_' + algorithm + filename_addition + '.npy', ratio_calibrated.classifier_.calibration_sample[:n_calibration_each]) elif t == settings.theta_benchmark_trained: np.save(results_dir + '/scores_trained_' + algorithm + '_calibrated' + filename_addition + '.npy', this_score) np.save(results_dir + '/r_trained_' + algorithm + '_calibrated' + filename_addition + '.npy', this_r) np.save(results_dir + '/r_vs_sm_trained_' + algorithm + '_calibrated' + filename_addition + '.npy', this_r / r_sm) np.save(results_dir + '/calvalues_trained_' + algorithm + filename_addition + '.npy', ratio_calibrated.classifier_.calibration_sample[:n_calibration_each]) ################################################################################ # Recalibration ################################################################################ if recalibration_mode: # Prepare data for recalibration thetas0_array = np.zeros((X_recalibration_transformed.shape[0], 2), dtype=X_recalibration_transformed.dtype) thetas0_array[:, :] = settings.thetas[t] X_thetas_recalibration = np.hstack((X_recalibration_transformed, thetas0_array)) # Evaluate recalibration data this_r, _ = ratio_calibrated.predict(X_thetas_recalibration) if t == settings.theta_observed: r_recalibration_sm = this_r recalibration_expected_r.append(np.mean(this_r / r_recalibration_sm)) ################################################################################ # Illustration ################################################################################ if t == settings.theta_benchmark_illustration: # Prepare data for illustration thetas0_array = np.zeros((X_illustration_transformed.shape[0], 2), dtype=X_illustration_transformed.dtype) thetas0_array[:, :] = settings.thetas[t] X_thetas_illustration = np.hstack((X_illustration_transformed, thetas0_array)) # Evaluate illustration data r_hat_illustration, _ = ratio_calibrated.predict(X_thetas_illustration) np.save(results_dir + '/r_illustration_' + algorithm + '_calibrated' + filename_addition + '.npy', r_hat_illustration) ################################################################################ # Neyman construction toys ################################################################################ # Neyman construction if do_neyman: # Prepare alternate data for Neyman construction thetas0_array = np.zeros((X_neyman_alternate_transformed.shape[0], 2), dtype=X_neyman_alternate_transformed.dtype) thetas0_array[:, :] = settings.thetas[t] X_thetas_neyman_alternate = np.hstack((X_neyman_alternate_transformed, thetas0_array)) # Neyman construction: alternate (calibrated) r_neyman_alternate, _ = ratio_calibrated.predict(X_thetas_neyman_alternate) llr_neyman_alternate = -2. * np.sum(np.log(r_neyman_alternate).reshape((-1, n_expected_events_neyman)), axis=1) np.save(neyman_dir + '/' + neyman_filename + '_llr_alternate_' + str( t) + '_' + algorithm + '_calibrated' + filename_addition + '.npy', llr_neyman_alternate) # Neyman construction: null X_neyman_null = np.load( settings.unweighted_events_dir + '/neyman/' + input_X_prefix + 'X_' + neyman_filename + '_null_' + str( t) + '.npy') X_neyman_null_transformed = scaler.transform( X_neyman_null.reshape((-1, X_neyman_null.shape[2]))) # Prepare null data for Neyman construction thetas0_array = np.zeros((X_neyman_null_transformed.shape[0], 2), dtype=X_neyman_null_transformed.dtype) thetas0_array[:, :] = settings.thetas[t] X_thetas_neyman_null = np.hstack((X_neyman_null_transformed, thetas0_array)) # Neyman construction: evaluate null (calibrated) r_neyman_null, _ = ratio_calibrated.predict(X_thetas_neyman_null) llr_neyman_null = -2. * np.sum( np.log(r_neyman_null).reshape((-1, n_expected_events_neyman)), axis=1) np.save(neyman_dir + '/' + neyman_filename + '_llr_null_' + str( t) + '_' + algorithm + '_calibrated' + filename_addition + '.npy', llr_neyman_null) # NC: null evaluated at alternate if t == settings.theta_observed: for tt in range(settings.n_thetas): X_neyman_null = np.load(settings.unweighted_events_dir + \ '/neyman/' + input_X_prefix + \ 'X_' + neyman_filename + \ '_null_' + str(tt) + '.npy') X_neyman_null_transformed = scaler.transform( X_neyman_null.reshape((-1, X_neyman_null.shape[2]))) X_thetas_neyman_null = np.hstack((X_neyman_null_transformed, thetas0_array)) # Neyman construction: evaluate null sample (calibrated) r_neyman_null, _ = ratio_calibrated.predict(X_thetas_neyman_null) llr_neyman_null = -2.*np.sum( np.log(r_neyman_null).reshape((-1, n_expected_events_neyman)), axis=1) np.save(neyman_dir + '/' + neyman_filename + \ '_llr_nullatalternate_' + str(tt) + '_' + \ algorithm + '_calibrated' + filename_addition + \ '.npy', llr_neyman_null) # Roaming thetas0_array = np.zeros((n_roaming, 2), dtype=X_roam_transformed.dtype) thetas0_array[:, :] = settings.thetas[t] X_thetas_roaming_temp = np.hstack((X_roam_transformed, thetas0_array)) r_roam_temp[t, :], _ = ratio_calibrated.predict(X_thetas_roaming_temp) # Save evaluation results expected_llr_calibrated = np.asarray(expected_llr_calibrated) mse_log_r_calibrated = np.asarray(mse_log_r_calibrated) trimmed_mse_log_r_calibrated = np.asarray(trimmed_mse_log_r_calibrated) expected_r_vs_sm = np.asarray(expected_r_vs_sm) if recalibration_mode: recalibration_expected_r = np.asarray(recalibration_expected_r) np.save(results_dir + '/llr_' + algorithm + '_calibrated' + \ filename_addition + '.npy', expected_llr_calibrated) np.save(results_dir + '/mse_logr_' + algorithm + '_calibrated' + \ filename_addition + '.npy', mse_log_r_calibrated) np.save(results_dir + '/trimmed_mse_logr_' + algorithm + \ '_calibrated' + filename_addition + '.npy', trimmed_mse_log_r_calibrated) np.save(results_dir + '/expected_r_vs_sm_' + algorithm + \ '_calibrated' + filename_addition + '.npy', expected_r_vs_sm) if recalibration_mode: recalibration_expected_r = np.asarray(recalibration_expected_r) np.save(results_dir + '/recalibration_expected_r_vs_sm_' + \ algorithm + '_calibrated' + filename_addition + '.npy', recalibration_expected_r) # Evaluation times logging.info('Calibrated evaluation timing: median %s s, mean %s s', np.median(eval_times), np.mean(eval_times)) logging.info('Interpolating calibrated roaming') gp = GaussianProcessRegressor(normalize_y=True, kernel=C(1.0) * Matern(1.0, nu=0.5), n_restarts_optimizer=10) gp.fit(settings.thetas[:], np.log(r_roam_temp)) r_roam_calibrated = np.exp(gp.predict(np.c_[xx.ravel(), yy.ravel()])).T np.save(results_dir + '/r_roam_' + algorithm + '_calibrated' + \ filename_addition + '.npy', r_roam_calibrated)
def run(args, mass, winLow, winHigh, bkghist, template, optKernel): GPh = GPHisto(bkghist) GPh.setWindow(winLow, winHigh) X = GPh.getXWindowArr() y = GPh.getYWindowArr() dy = GPh.getErrWindowArr() # The distributions with no window removed. X_t = GPh.getXArr() y_t = GPh.getYArr() dy_t = GPh.getErrArr() if args.noWindow: X = X_t y = y_t dy = dy_t x = GPh.getXArr() gp = None if optKernel is None: kernel = C(10.0, (1e-3, 1e15)) * RBF( 50.0, (1e-3, 1e6)) #squared exponential kernel #kernel = C(1000.0, (1e-3, 1e15)) * FallExp(1.0, (1e-5, 1e2), 1.0, (1e-3,1e15)) * Gibbs(1.0, (1e-3, 1e5), 1.0, (1e-3,1e5)) gp = GaussianProcessRegressor( kernel=kernel #,optimizer=None , alpha=dy**2, n_restarts_optimizer=9) else: kernel = optKernel gp = GaussianProcessRegressor(kernel=kernel, optimizer=None, alpha=dy**2, n_restarts_optimizer=9) gp.fit(X, y) y_pred, sigma = gp.predict(x, return_std=True) outhist = GPh.getHisto(y_pred, 1.96 * sigma, 'GP Fit') print "outhist nbins:", outhist.GetNbinsX() if args.noWindow: bkgWindow = GPh.getHisto(y, dy, 'Full Background') else: bkgWindow = GPh.getWinHisto(y, dy, 'Full Background') errCov = outhist.Clone('errorCoverage') errCov.Reset() norm = (bkghist.Integral() / template.Integral()) print gp.kernel_ for nbin in range(1, outhist.GetNbinsX()): Err = outhist.GetBinError(nbin) cont = outhist.GetBinContent(nbin) binCenter = outhist.GetBinCenter(nbin) temp = norm * template.GetBinContent(template.FindBin(binCenter)) if temp <= (cont + Err) and temp >= (cont - Err): errCov.Fill(binCenter) else: print "found uncovered bin: ", nbin print "LowErr : template : HighErr ---- {0} : {1} : {2}".format( (cont - Err), temp, (cont + Err)) print "----------" return errCov pass #run
import numpy as np from scipy.optimize import fmin_l_bfgs_b from scipy.stats import uniform import matplotlib.pyplot as plt from sklearn.gaussian_process.kernels import Matern, ConstantKernel as C from bolero.environment.catapult import Catapult from bolero_bayes_opt import BOPSOptimizer catapult = Catapult([(0, 0), (2.0, -0.5), (3.0, 0.5), (4, 0.25), (5, 2.5), (7, 0), (10, 0.5), (15, 0)]) catapult.init() kernel = C(100.0, (1.0, 10000.0)) \ * Matern(length_scale=(1.0, 1.0), length_scale_bounds=[(0.1, 100), (0.1, 100)]) opt = BOPSOptimizer(boundaries=[(5, 10), (0, np.pi / 2)], bo_type="bo", acquisition_function="UCB", acq_fct_kwargs=dict(kappa=2.5), optimizer="direct+lbfgs", maxf=100, gp_kwargs=dict(kernel=kernel, normalize_y=True, alpha=1e-5)) target = 4.0 # Fixed target context = (target - 2.0) / 8.0 # Compute maximal achievable reward for this target optimal_reward = -np.inf
def gomez_bombarelli_opt(X_train, pred_vae, ground_truth, total_it=10000, constrained=True): """Runs the Gomez-Bombarelli optimization algorithm""" LD = 20 L = X_train.shape[1] embeddings = pred_vae.encoder_.predict([X_train])[0] predictions = pred_vae.predictor_.predict(embeddings)[0] predictions = predictions.reshape((predictions.shape[0])) kernel = C(1.0, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2)) gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=0, normalize_y=True) print("Fitting GP...") gp.fit(embeddings, predictions) print("Finishing fitting GP") def gp_z_estimator(z): val = gp.predict(z.reshape(1, -1)).ravel()[0] return -val method = 'COBYLA' num_it = 0 f_opts = [] gt_opts = [] z0 = np.random.randn(LD) if constrained: z_norm = np.linalg.norm(embeddings, axis=1) z_norm_mean = np.mean(z_norm) z_norm_std = np.std(z_norm) lower_rad = z_norm_mean - z_norm_std * 2 higher_rad = z_norm_mean + z_norm_std * 2 constraints = [{ 'type': 'ineq', 'fun': lambda x: np.linalg.norm(x) - lower_rad }, { 'type': 'ineq', 'fun': lambda x: higher_rad - np.linalg.norm(x) }] res = minimize(gp_z_estimator, z0, method=method, constraints=constraints, options={ 'disp': False, 'tol': 0.1, 'maxiter': 10000 }, tol=0.1) else: res = minimize(gp_z_estimator, z0, method=method, options={ 'disp': False, 'tol': 0.1, 'maxiter': 10000 }, tol=0.1) z_opt = res.x.reshape((1, LD)) f_opt = -res.fun x_opt = np.argmax(pred_vae.decoder_.predict(z_opt), axis=-1) gt_opt = ground_truth.predict(x_opt)[:, 0][0] print(f_opt, gt_opt) oracle_max_seq = util.convert_idx_array_to_aas(x_opt) max_dict = { 'oracle_max': f_opt, 'oracle_max_seq': oracle_max_seq, 'gt_of_oracle_max': gt_opt } results = np.array([f_opt, gt_opt]) return results, max_dict
def main(self): self.x = [-4, 4, 5] self.y = [-4, 4, 5] self.xy, shape1 = self.create_parameter_space([self.x, self.y]) xy0 = self.xy # Save initial grid for sampling z = [] for i in range(np.shape(self.xy)[0]): z.append(self.f(self.xy[i][0], self.xy[i][1])) self.z = np.array(z) # print(self.parameter_space_shape) # print(xy[:,0]) # print(xy[0,:]) # print(np.shape(xy)) # print(np.shape(z)) # Add noise: dz = 0.01 + 0.02 * np.random.random(self.z.shape) noise = np.random.normal(0, dz) self.z += noise # Instantiate a Gaussian Process model kernel = C(1.0, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2)) + WhiteKernel( ) # This seems to work and is what Robbie suggested # kernel = DotProduct() + WhiteKernel() # kernel = 1.0 * RBF(1.0) + 1.0 * WhiteKernel() # this doesn't give an equivalent output to None. Not sure why. # kernel = None self.gp = GaussianProcessRegressor( kernel=kernel, n_restarts_optimizer=9 ) # None is passed, the kernel “1.0 * RBF(1.0)” is used as default. # Fit to data using Maximum Likelihood Estimation of the parameters print("xy: ", np.shape(self.xy)) print("z: ", np.shape(self.z)) self.gp.fit(self.xy, self.z) # Make the prediction on the meshed x-axis (ask for MSE as well) z_preds, sigma = self.gp.predict(self.xy, return_std=True) # self.plot(self.z, z_preds, self.xy, shape1, plot_residuals = True) self.calculate_avg_residual(self.z, z_preds) print("Initial min: %f" % min(z_preds)) xx = [-4, 4, 15] yy = [-4, 4, 15] xy_dense, shape2 = self.create_parameter_space([xx, yy]) zdense = [] for i in range(np.shape(xy_dense)[0]): zdense.append(self.f(xy_dense[i][0], xy_dense[i][1])) zdense = np.array(zdense) zdense_preds, sigma = self.gp.predict(xy_dense, return_std=True) # self.plot(zdense, zdense_preds, xy_dense, shape2, plot_residuals = True) self.calculate_avg_residual(zdense, zdense_preds) print("Initial min on dense mesh: %f" % min(zdense_preds)) self.min_search() zdense_preds, sigma = self.gp.predict( xy_dense, return_std=True) # Replot things after the minsearch # self.plot(zdense, zdense_preds, xy_dense, shape2, plot_residuals = True) self.calculate_avg_residual(zdense, zdense_preds) print("Min on dense mesh after training: %f" % min(zdense_preds)) self.min_search() zdense_preds, sigma = self.gp.predict( xy_dense, return_std=True) # Replot things after the minsearch # self.plot(zdense, zdense_preds, xy_dense, shape2, plot_residuals = True) self.calculate_avg_residual(zdense, zdense_preds) print("Min on dense mesh after training: %f" % min(zdense_preds)) self.min_search() zdense_preds, sigma = self.gp.predict( xy_dense, return_std=True) # Replot things after the minsearch # self.plot(zdense, zdense_preds, xy_dense, shape2, plot_residuals = True) self.calculate_avg_residual(zdense, zdense_preds) print("Min on dense mesh after training: %f" % min(zdense_preds))
samplePoints = 10000 #how many random measurements of the function will be done before calculating expected improvement no_startingPoints = 100 # tavalist = 100 suurem = 10000 #algpunktide genereerimine X = np.random.uniform(bounds[0, 0], bounds[0, 1], [no_startingPoints, 1]) Y = np.random.uniform(bounds[1, 0], bounds[1, 1], [no_startingPoints, 1]) XY = np.column_stack((X, Y)) Z = tf.rosenbrock(XY, 1, 100) #Mis mudel ja Kernel customKernel = C(1.0) * Matern() model = GaussianProcessRegressor(kernel=customKernel) for i in range(n_iterations): #print iteration print("Iteration " + str(i) + "/" + str(n_iterations)) # Update Gaussian process with existing samples model.fit(XY, Z) # Obtain next sampling point from the acquisition function (expected_improvement) XY_next = bf.propose_location(bf.expected_improvement, XY, Z, model, bounds, samplePoints, tavalist, suurem, no_startingPoints, exploration) # Obtain next sample from the objective function
# ### 4. Busca los mejores hiperparámetros del GP para predecir la serie temporal del CO2 usando datos hasta la fecha más reciente. Compara estos hiperparámetros con los que se encontraron al usar datos hasta diciembre de 2001 (los datos usados en la práctica). # In[6]: from sklearn.gaussian_process.kernels import RationalQuadratic from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C, WhiteKernel, ExpSineSquared from time import time # Kernel with generic parameters k1 = C(50, (1e1, 1e+3)) * RBF(50,(1e0,1e5)) # long term smooth rising trend k2 = C(5, (1e1, 1e+2)) * RBF(100,(1e0,1e5)) * ExpSineSquared(length_scale=1.0, length_scale_bounds=(1e-1,1e1), periodicity=1.0, periodicity_bounds="fixed") # seasonal component k3 = C(0.5, (1e-2, 1e1)) * RationalQuadratic(length_scale=1.0, length_scale_bounds=(1e-2,1e3), alpha=1.0, alpha_bounds=(1e-1,1e3)) # medium term irregularities k4 = C(0.1, (1e-3, 1e+1)) * RBF(0.1,(1e-2,1e2)) + WhiteKernel(noise_level=0.1**2,noise_level_bounds=(1e-4, 1e0)) # noise terms kernel = k1 + k2 + k3 + k4 # Fit the Gaussian Process regressor, and optimize the hyperparameters gp = GaussianProcessRegressor(kernel=kernel, alpha=0, normalize_y=True, n_restarts_optimizer=1) t0 = time() # In[7]:
ystar, sigma = gpr.predict(x, return_std=True) theta = gpr.kernel_.theta #lml = gpr.log_marginal_likelihood(theta=theta) lml = gpr.log_marginal_likelihood() sigma = np.reshape(sigma, (np.size(sigma), 1)) sigma = (sigma**2 + 1)**0.5 ystarp = ystar + sigma ystari = scaler.inverse_transform(ystar) ystarpi = scaler.inverse_transform(ystarp) sigmai = (ystarpi - ystari) return ystari, sigmai, gpr, theta, lml k1_iwu = np.random.uniform(1e-1, 1e1) k2_iwu = np.random.uniform(1e-1, 1e1) kernel_standard_gp = C(k1_iwu, (1e-1, 1e1)) * RBF(k2_iwu, (1e-1, 1e1)) start = time.time() ypred_sk_weak, sigma_sk_weak, _, theta, lml = GP_train_kernel( x, snr, kernel_standard_gp) end = time.time() #np.savetxt('ypred_sk_weak.csv', ypred_sk, delimiter=',') #np.savetxt('sigma_sk_weak.csv', sigma_sk, delimiter=',') print("weak prior") print("GP fitting took " + str((end - start) / 60) + " minutes") print("LML = " + str(lml)) thetatrans = np.exp(theta) print("theta (linear) = " + str(thetatrans)) print("theta (log) = " + str(theta)) print("sigma mean = " + str(np.mean(sigma_sk_weak))) kernel_standard_gp = C(k1_o, (k1_o / 1000, k1_o * 1000)) * RBF(
def GP(lc, restarts=0, kernel_type=1): bands = np.unique(lc.photometry['lambda_cen']) X = np.vstack([ lc.photometry['mjd'] - np.min(lc.photometry['mjd']), lc.photometry['lambda_cen'] ]).T y = lc.photometry['flux'] - np.min(lc.photometry['flux']) dy = lc.photometry['fluxerr'] if kernel_type == 0: ### Fit a 9-parameter (3x3 RBF) kernel kn = kernel_RBF() final_gp = GaussianProcessRegressor(kernel=kn, alpha=dy, n_restarts_optimizer=restarts) final_gp.fit(X, y) kps = final_gp.kernel_.get_params() kernel_params = [kps['k1__k1__k1'].constant_value,\ kps['k1__k1__k2'].length_scale[0],\ kps['k1__k1__k2'].length_scale[1],\ kps['k1__k2__k1'].constant_value,\ kps['k1__k2__k2'].length_scale[0],\ kps['k1__k2__k2'].length_scale[1],\ kps['k2__k1'].constant_value,\ kps['k2__k2'].length_scale[0],\ kps['k2__k2'].length_scale[1],\ final_gp.log_marginal_likelihood(),\ ] elif kernel_type == 1: ### Iteratively fit to the residuals k1 = kernel_RBF_general(5e3, 6e3, 1e0, 1e6, 1e0, 1e6) gp1 = GaussianProcessRegressor(kernel=k1, alpha=dy, n_restarts_optimizer=restarts) gp1.fit(X, y) kp1 = gp1.kernel_.get_params() gp1_result = gp1.predict(X) ### Fit the residuals to an RBF with smaller length scales k2 = kernel_RBF_general(1e2, 3e2, 1e0, kp1['length_scale'][0], 1e0, kp1['length_scale'][1]) gp2 = GaussianProcessRegressor(kernel=k2, alpha=dy, n_restarts_optimizer=restarts) gp2.fit(X, y - gp1_result) kp2 = gp2.kernel_.get_params() gp2_result = gp2.predict(X) ### Fit the second residuals to a final RBF with the smallest length scales (trying to capture noise). k3 = kernel_RBF_general(5e0, 3e1, 1e0, kp2['length_scale'][0], 1e0, kp2['length_scale'][1]) gp3 = GaussianProcessRegressor(kernel=k3, alpha=dy, n_restarts_optimizer=restarts) gp3.fit(X, y - gp1_result - gp2_result) kp3 = gp3.kernel_.get_params() final_kernel = C(1.,(1e-2,1e3)) * RBF((kp1['length_scale'][0],kp1['length_scale'][1])) +\ C(1.,(1e-2,1e3)) * RBF((kp2['length_scale'][0],kp2['length_scale'][1])) +\ C(1.,(1e-2,1e3)) * RBF((kp3['length_scale'][0],kp3['length_scale'][1])) final_gp = GaussianProcessRegressor(kernel=final_kernel, alpha=dy, n_restarts_optimizer=restarts) final_gp.fit(X, y) kps = final_gp.kernel_.get_params() kernel_params = [kps['k1__k1__k1'].constant_value,\ kps['k1__k1__k2'].length_scale[0],\ kps['k1__k1__k2'].length_scale[1],\ kps['k1__k2__k1'].constant_value,\ kps['k1__k2__k2'].length_scale[0],\ kps['k1__k2__k2'].length_scale[1],\ kps['k2__k1'].constant_value,\ kps['k2__k2'].length_scale[0],\ kps['k2__k2'].length_scale[1],\ final_gp.log_marginal_likelihood(),\ ] elif kernel_type == 2: kn = kernel_Matern() final_gp = GaussianProcessRegressor(kernel=kn, alpha=dy, n_restarts_optimizer=restarts) final_gp.fit(X, y) kps = final_gp.kernel_.get_params() kernel_params = [kps['k1'].constant_value,\ kps['k2'].length_scale[0],\ kps['k2'].length_scale[1],\ final_gp.log_marginal_likelihood(),\ ] else: raise ValueError('Parameter "kernel_type" must be one of [0,1,2].') # generate realization of GP fit on the data time_real = np.arange(np.floor(np.min(X[:, 0])), np.ceil(np.max(X[:, 0])) + 1.) tgp, wgp = np.meshgrid(time_real, bands) x_real = np.vstack([tgp.flatten(), wgp.flatten()]).T y_real = final_gp.predict(x_real) y_real = y_real.reshape(tgp.shape).T # compute features from the realization of the fit # Peak magnitude of the GP flux prediction in the LSST i band pkmag_i = 22.0 - 2.5 * np.log10(np.max(y_real[:, 3])) # Ratio of the maximum positive flux to the maximum-minus-minimum flux in the LSST i band pos_flux_ratio = np.max( y_real[:, 3]) / (np.max(y_real[:, 3]) - np.min(y_real[:, 3])) # Normalized difference of the light curve colors at maximum/minimum light. The blue # measurement is the difference between the LSST i and g bands, and the red measurement # is the difference between the LSST y and i bands. The normalized difference is calculated # by taking the difference of the fluxes in the two bands divided by their sum. max_light_idx = np.where(y_real == np.max(y_real))[0][0] min_light_idx = np.where(y_real == np.min(y_real))[0][0] # max_fr_blue = (y_real[max_light_idx, 3] - y_real[max_light_idx, 1]) / ( y_real[max_light_idx, 3] + y_real[max_light_idx, 1]) min_fr_blue = (y_real[min_light_idx, 3] - y_real[min_light_idx, 1]) / ( y_real[min_light_idx, 3] + y_real[min_light_idx, 1]) max_fr_red = (y_real[max_light_idx, 5] - y_real[max_light_idx, 3]) / ( y_real[max_light_idx, 5] + y_real[max_light_idx, 3]) min_fr_red = (y_real[min_light_idx, 5] - y_real[min_light_idx, 3]) / ( y_real[min_light_idx, 5] + y_real[min_light_idx, 3]) # Difference of the time of maximum in the LSST y and g bands in days max_dt_yg = time_real[np.argmax(y_real[:, 5])] - time_real[np.argmax( y_real[:, 1])] # An estimate of the light curve “width” that is applicable even for non-supernova-like # transients and variables. This is implemented as the integral of the positive/negative # parts of the GP flux predictions divided by the positive/negative maximum fluxes. positive_width = integrate.trapz(y_real[:, 3] - np.median(y_real[:, 3])) / np.max( y_real[:, 3]) # Measurements of the rise and decline times of a light curve. This measurement is defined # as the time in days for the light curve to rise (bwd) or decline (fwd) to a given fraction # (either 20% or 50%) of maximum light in the LSST i band. ### NOTE: REVISIT THE DEFAULT EXCEPTION VALUES HERE (taken to be 1) ### REPLACE WITH MEAN VALUES try: time_fwd_max_20 = np.where(np.logical_and(\ y_real[:,3] <= (0.2 * (np.max(y_real[:,3]) - np.min(y_real[:,3])) + np.min(y_real[:,3])),\ time_real > np.argmax(y_real[:,3])\ ))[0][0] - np.argmax(y_real[:,3]) except IndexError: time_fwd_max_20 = 1. try: time_fwd_max_50 = np.where(np.logical_and(\ y_real[:,3] <= (0.5 * (np.max(y_real[:,3]) - np.min(y_real[:,3])) + np.min(y_real[:,3])),\ time_real > np.argmax(y_real[:,3])\ ))[0][0] - np.argmax(y_real[:,3]) except IndexError: time_fwd_max_50 = 1. try: time_bwd_max_20 = np.argmax(y_real[:,3]) - np.where(np.logical_and(\ y_real[:,3] <= (0.2 * (np.max(y_real[:,3]) - np.min(y_real[:,3])) + np.min(y_real[:,3])),\ time_real < np.argmax(y_real[:,3])\ ))[0][-1] except IndexError: time_bwd_max_20 = 1. try: time_bwd_max_50 = np.argmax(y_real[:,3]) - np.where(np.logical_and(\ y_real[:,3] <= (0.5 * (np.max(y_real[:,3]) - np.min(y_real[:,3])) + np.min(y_real[:,3])),\ time_real < np.argmax(y_real[:,3])\ ))[0][-1] except IndexError: time_bwd_max_50 = 1. # Ratio of the rise/decline times calculated as described above in different bands. The blue # measurement is the difference between the LSST i and g bands, and the red measurement # is the difference between the LSST y and i bands. try: time_fwd_max_20_ratio_blue = float(time_fwd_max_20 / ((np.where( np.logical_and( y_real[:, 1] <= (0.2 * (np.max(y_real[:, 1]) - np.min(y_real[:, 1])) + np.min(y_real[:, 1])), time_real > np.argmax(y_real[:, 1]))) [0][0])) - np.argmax(y_real[:, 1])) except IndexError: time_fwd_max_20_ratio_blue = 1. #float(time_fwd_max_20 / int(time_real[-1])) try: time_fwd_max_50_ratio_blue = float(time_fwd_max_50 / ((np.where( np.logical_and( y_real[:, 1] <= (0.5 * (np.max(y_real[:, 1]) - np.min(y_real[:, 1])) + np.min(y_real[:, 1])), time_real > np.argmax(y_real[:, 1]))) [0][0])) - np.argmax(y_real[:, 1])) except IndexError: time_fwd_max_50_ratio_blue = 1. #float(time_fwd_max_50 / int(time_real[-1])) try: time_bwd_max_20_ratio_blue = float( time_bwd_max_20 / (np.argmax(y_real[:, 1]) - (np.where( np.logical_and( y_real[:, 1] <= (0.2 * (np.max(y_real[:, 1]) - np.min(y_real[:, 1])) + np.min(y_real[:, 1])), time_real < np.argmax(y_real[:, 1])))[0][-1]))) except IndexError: time_bwd_max_20_ratio_blue = 1. #float(time_bwd_max_20 / int(time_real[1])) try: time_bwd_max_50_ratio_blue = float( time_bwd_max_50 / (np.argmax(y_real[:, 1]) - (np.where( np.logical_and( y_real[:, 1] <= (0.5 * (np.max(y_real[:, 1]) - np.min(y_real[:, 1])) + np.min(y_real[:, 1])), time_real < np.argmax(y_real[:, 1])))[0][-1]))) except IndexError: time_bwd_max_50_ratio_blue = 1. #float(time_bwd_max_50 / int(time_real[1])) try: time_fwd_max_20_ratio_red = float(((np.where( np.logical_and( y_real[:, 5] <= (0.2 * (np.max(y_real[:, 5]) - np.min(y_real[:, 5])) + np.min(y_real[:, 5])), time_real > np.argmax( y_real[:, 1])))[0][0]) - np.argmax(y_real[:, 5])) / time_fwd_max_20) except IndexError: time_fwd_max_20_ratio_red = 1. #float(int(time_real[-1]) / time_fwd_max_20) try: time_fwd_max_50_ratio_red = float(((np.where( np.logical_and( y_real[:, 5] <= (0.5 * (np.max(y_real[:, 5]) - np.min(y_real[:, 5])) + np.min(y_real[:, 5])), time_real > np.argmax( y_real[:, 5])))[0][0]) - np.argmax(y_real[:, 5])) / time_fwd_max_50) except IndexError: time_fwd_max_50_ratio_red = 1. #float(int(time_real[-1]) / time_fwd_max_50) try: time_bwd_max_20_ratio_red = float((np.argmax(y_real[:, 5]) - (np.where( np.logical_and( y_real[:, 5] <= (0.2 * (np.max(y_real[:, 5]) - np.min(y_real[:, 5])) + np.min(y_real[:, 5])), time_real < np.argmax(y_real[:, 5])))[0][-1])) / time_bwd_max_20) except IndexError: time_bwd_max_20_ratio_red = 1. #float(int(time_real[0]) / time_bwd_max_20) try: time_bwd_max_50_ratio_red = float((np.argmax(y_real[:, 5]) - (np.where( np.logical_and( y_real[:, 5] <= (0.5 * (np.max(y_real[:, 5]) - np.min(y_real[:, 5])) + np.min(y_real[:, 5])), time_real < np.argmax(y_real[:, 5])))[0][-1])) / time_bwd_max_50) except IndexError: time_bwd_max_50_ratio_red = 1. #float(int(time_real[0]) / time_bwd_max_50) # Fraction of observations that have a signal greater than 5/less than -5 times the noise level. frac_s2n_5 = len(np.where(lc.photometry['SNR'] > 5.)[0]) / len( lc.photometry['SNR']) # Fraction of observations that have an absolute signal-to-noise less than 3 frac_background = len(np.where( np.abs(lc.photometry['SNR']) < 3.)[0]) / len(lc.photometry['SNR']) # Time difference in days between the first observation with a signal-to-noise greater than 5 # and the last such observation (in any band) time_width_s2n_5 = lc.photometry['mjd'][np.where( lc.photometry['SNR'] > 5.)[0][-1]] - lc.photometry['mjd'][np.where( lc.photometry['SNR'] > 5.)[0][0]] # Number of observations in any band within 5 days of maximum light max_light_mjd = np.min(lc.photometry['mjd']) + max_light_idx # count_max_center = len( np.where( np.logical_and(lc.photometry['mjd'] < 5. + max_light_mjd, lc.photometry['mjd'] > max_light_mjd - 5.))[0]) # Number of observations in any band between 20, 50, or 100 days before maximum light # and 5 days after maximum light. count_max_rise_20 = len( np.where( np.logical_and(lc.photometry['mjd'] < 5. + max_light_mjd, lc.photometry['mjd'] > max_light_mjd - 20.))[0]) count_max_rise_50 = len( np.where( np.logical_and(lc.photometry['mjd'] < 5. + max_light_mjd, lc.photometry['mjd'] > max_light_mjd - 50.))[0]) count_max_rise_100 = len( np.where( np.logical_and(lc.photometry['mjd'] < 5. + max_light_mjd, lc.photometry['mjd'] > max_light_mjd - 100.))[0]) # Number of observations in any band between 5 days before maximum light and 20, 50, or # 100 days after maximum light. count_max_fall_20 = len( np.where( np.logical_and(lc.photometry['mjd'] < 20. + max_light_mjd, lc.photometry['mjd'] > max_light_mjd - 5.))[0]) count_max_fall_50 = len( np.where( np.logical_and(lc.photometry['mjd'] < 50. + max_light_mjd, lc.photometry['mjd'] > max_light_mjd - 5.))[0]) count_max_fall_100 = len( np.where( np.logical_and(lc.photometry['mjd'] < 100. + max_light_mjd, lc.photometry['mjd'] > max_light_mjd - 5.))[0]) # Total signal-to-noise of all observations of the object. total_s2n = np.sum(lc.photometry['SNR']) fit_params = [pkmag_i, pos_flux_ratio, max_fr_blue, min_fr_blue, max_fr_red, min_fr_red, max_dt_yg, positive_width,\ time_fwd_max_20, time_fwd_max_50, time_bwd_max_20, time_bwd_max_50, time_fwd_max_20_ratio_blue,\ time_fwd_max_50_ratio_blue, time_bwd_max_20_ratio_blue, time_bwd_max_50_ratio_blue,\ time_fwd_max_20_ratio_red, time_fwd_max_50_ratio_red, time_bwd_max_20_ratio_red,\ time_bwd_max_50_ratio_red, frac_s2n_5, frac_background, time_width_s2n_5,\ count_max_center, count_max_rise_20, count_max_rise_50, count_max_rise_100,\ count_max_fall_20, count_max_fall_50, count_max_fall_100,\ total_s2n] return kernel_params, fit_params
num_of_episodes = 30 q = {} states = [(x, y) for x in range(-GRID, GRID + 1) for y in range(-GRID, GRID + 1)] actions = [(1, 0), (-1, 0), (0, 1), (0, -1)] term_state = (GRID, GRID) # train_data_f = [[2, 3], [5, 6]] # train_data_t = [[7], [8]] # mean = np.zeros(train_data_f.shape) kernel = C(10.0, (1e-3, 1e3)) * RBF(0.5, (1e-2, 1e2)) + WhiteKernel(0.01, (1e-5, 1e5)) # sampled = np.random.multivariate_normal( np.squeeze(mean), kernel(train_data_f)) # gp.fit(train_data_f, train_data_t) # plt.plot(data, np.sin(data)) # plt.show() # print gp.get_params(deep=True) # x = np.arange(-10 , 10, 0.2)[:, np.newaxis] # print y_pred # plt.scatter(train_data_f, train_data_t)
def test_gpr_rbf_unfitted(self): se = (C(1.0, (1e-3, 1e3)) * RBF(length_scale=10, length_scale_bounds=(1e-3, 1e3))) kernel = (Sum( se, C(0.1, (1e-3, 1e3)) * RBF(length_scale=1, length_scale_bounds=(1e-3, 1e3)))) gp = GaussianProcessRegressor(alpha=1e-7, kernel=kernel, n_restarts_optimizer=15, normalize_y=True) # return_cov=False, return_std=False model_onnx = to_onnx(gp, initial_types=[('X', FloatTensorType([]))], dtype=np.float32, target_opset=_TARGET_OPSET_) self.assertTrue(model_onnx is not None) dump_data_and_model(Xtest_.astype(np.float32), gp, model_onnx, verbose=False, basename="SklearnGaussianProcessRBFUnfitted") # return_cov=True, return_std=True options = { GaussianProcessRegressor: { "return_std": True, "return_cov": True } } try: to_onnx(gp, Xtrain_.astype(np.float32), options=options, target_opset=TARGET_OPSET) except RuntimeError as e: assert "Not returning standard deviation" in str(e) # return_std=True options = {GaussianProcessRegressor: {"return_std": True}} model_onnx = to_onnx(gp, options=options, initial_types=[('X', FloatTensorType([None, None]))], dtype=np.float32, target_opset=TARGET_OPSET) self.assertTrue(model_onnx is not None) self.check_outputs( gp, model_onnx, Xtest_.astype(np.float32), predict_attributes=options[GaussianProcessRegressor]) # return_cov=True options = {GaussianProcessRegressor: {"return_cov": True}} # model_onnx = to_onnx(gp, Xtrain_.astype(np.float32), options=options) model_onnx = to_onnx(gp, options=options, initial_types=[('X', FloatTensorType([None, None]))], dtype=np.float32, target_opset=TARGET_OPSET) self.assertTrue(model_onnx is not None) self.check_outputs( gp, model_onnx, Xtest_.astype(np.float32), predict_attributes=options[GaussianProcessRegressor])
"""The function to predict (classification will then consist in predicting whether g(x) <= 0 or not)""" return 5. - x[:, 1] - .5 * x[:, 0]**2. # Design of experiments X = np.array([[-4.61611719, -6.00099547], [4.10469096, 5.32782448], [0.00000000, -0.50000000], [-6.17289014, -4.6984743], [1.3109306, -6.93271427], [-5.03823144, 3.10584743], [-2.87600388, 6.74310541], [5.21301203, 4.26386883]]) # Observations y = np.array(g(X) > 0, dtype=int) # Instanciate and fit Gaussian Process Model kernel = C(0.1, (1e-5, np.inf)) * DotProduct(sigma_0=0.1)**2 gp = GaussianProcessClassifier(kernel=kernel) gp.fit(X, y) print("Learned kernel: %s " % gp.kernel_) # Evaluate real function and the predicted probability res = 50 x1, x2 = np.meshgrid(np.linspace(-lim, lim, res), np.linspace(-lim, lim, res)) xx = np.vstack([x1.reshape(x1.size), x2.reshape(x2.size)]).T y_true = g(xx) y_prob = gp.predict_proba(xx)[:, 1] y_true = y_true.reshape((res, res)) y_prob = y_prob.reshape((res, res)) # Plot the probabilistic classification iso-values
# save history to a csv file if save_history: history_x = self.res.x_iters[0:self.n_calls] if self.flag_maximize: history_y = self.res.func_vals[0:self.n_calls] * (-1) else: history_y = self.res.func_vals[0:self.n_calls] history_y_list = history_y.tolist() history_x_list = history_x with open(self.save_path + "SoranoSat_History_{0}.csv".format(self.date), "w") as f: f.write(columns_name_list_str) for X, Y in zip(history_x_list, history_y_list): X.append(Y) buf_list = list(map(str, X)) buf_str = ','.join(buf_list) f.write(buf_str + '\n') print("***** Optimization finished ({0}) *****".format(str(self.idx))) if __name__ == "__main__": _data_path = "data/SoranoSat_Data.csv" iteration = 3 for i in range(iteration): BO = BayesianOptimization(data_path=_data_path, n_calls=10, idx=i) BO.preprocess() _kernel = C(1.0, (1e-2, 1e2)) * RBF(1.0, (1e-2, 1e2)) + Wh(0.01, (1e-2, 1e2)) BO.gaussian_process(kernel=_kernel, save_model=True) BO.run_optimization() BO.save_result(save_history=False) cut_table(data_path=_data_path, line_to_cut_off=iteration)
bkghist_template = f.Get('hmgg_c0') template = f.Get('hmgg_c0') bkghist_template.Rebin(8) stats = 60000 optKernel = None GPh = GPHisto(bkghist_template) #GPh.setWindow(winLow,winHigh) # The distributions with no window removed. X = GPh.getXArr() y = GPh.getYArr() dy = GPh.getErrArr() x = GPh.getXArr() kernel = C(10.0, (1e-3, 1e15)) * RBF(50.0, (1e-3, 1e6)) #squared exponential kernel #kernel = C(1000.0, (1e-3, 1e15)) * FallExp(1.0, (1e-5, 1e2), 1.0, (1e-3,1e15)) * Gibbs(1.0, (1e-3, 1e5), 1.0, (1e-3,1e5)) gp = GaussianProcessRegressor( kernel=kernel #,optimizer=None , alpha=dy**2, n_restarts_optimizer=9) gp.fit(X, y) print gp.kernel_ raise () optKernel = gp.kernel_
def train(regress_type='hybrid', seed=1, **kwargs): X_obs = kwargs['X_obs'] y_obs = kwargs['y_obs'] Kds = kwargs['Kds'] kwargs['regress_type'] = regress_type # Debug. #X_obs = X_obs[:10] #y_obs = y_obs[:10] # Fit the model. if regress_type == 'baseline': from baseline import Baseline regressor = Baseline() elif regress_type == 'mlper1': regressor = mlp_ensemble( n_neurons=200, n_regressors=1, n_epochs=50, seed=seed, ) elif regress_type == 'dmlper1': regressor = mlp_ensemble( n_neurons=1000, n_regressors=1, n_hidden_layers=15, n_epochs=300, seed=seed, ) elif regress_type == 'mlper1g': regressor = mlp_ensemble( n_neurons=100, n_regressors=1, n_epochs=100, loss='gaussian_nll', seed=seed, ) elif regress_type == 'mlper5': regressor = mlp_ensemble( n_neurons=200, n_regressors=5, n_epochs=100, seed=seed, ) elif regress_type == 'mlper5g': regressor = mlp_ensemble( n_neurons=200, n_regressors=5, n_epochs=50, loss='gaussian_nll', seed=seed, ) elif regress_type == 'bayesnn': from bayesian_neural_network import BayesianNN regressor = BayesianNN( n_hidden1=200, n_hidden2=200, n_iter=1000, n_posterior_samples=100, random_state=seed, verbose=True, ) elif regress_type == 'cmf': from cmf_regressor import CMFRegressor regressor = CMFRegressor( n_components=30, seed=seed, ) regressor.fit( kwargs['chems'], kwargs['prots'], kwargs['chem2feature'], kwargs['prot2feature'], kwargs['Kds'], kwargs['idx_obs'], ) elif regress_type == 'gp': from gaussian_process import GPRegressor from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C regressor = GPRegressor( kernel=C(10000., 'fixed') * RBF(1., 'fixed'), backend='sklearn', n_jobs=10, verbose=True ) elif regress_type == 'gpfactorized': from gaussian_process import GPRegressor from sklearn.gaussian_process.kernels import ConstantKernel as C from kernels import FactorizedRBF n_features_chem = kwargs['n_features_chem'] n_features_prot = kwargs['n_features_prot'] regressor = GPRegressor( kernel=C(1., 'fixed') * FactorizedRBF( [ 1.1, 1. ], [ n_features_chem, n_features_prot ], 'fixed' ), backend='sklearn', n_restarts=0, n_jobs=10, verbose=True ) elif regress_type == 'sparsegp': from gaussian_process import SparseGPRegressor regressor = SparseGPRegressor( method='geosketch', n_inducing=8000, backend='sklearn', n_restarts=10, n_jobs=10, verbose=True ) elif regress_type == 'hybrid': from gaussian_process import GPRegressor from hybrid import HybridMLPEnsembleGP regressor = HybridMLPEnsembleGP( mlp_ensemble( n_neurons=200, n_regressors=1, n_epochs=50, seed=seed, ), GPRegressor( backend='sklearn',#'gpytorch', n_restarts=10, n_jobs=10, verbose=True, ), ) elif regress_type == 'dhybrid': from gaussian_process import GPRegressor from hybrid import HybridMLPEnsembleGP regressor = HybridMLPEnsembleGP( mlp_ensemble( n_neurons=1000, n_regressors=1, n_hidden_layers=15, n_epochs=300, seed=seed, ), GPRegressor( backend='sklearn',#'gpytorch', n_restarts=10, n_jobs=10, verbose=True, ), ) elif regress_type == 'sparsehybrid': from gaussian_process import SparseGPRegressor from hybrid import HybridMLPEnsembleGP from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C regressor = HybridMLPEnsembleGP( mlp_ensemble( n_neurons=200, n_regressors=1, n_epochs=50, seed=seed, ), SparseGPRegressor( method='geosketch', n_inducing=8000, backend='sklearn', n_restarts=10, n_jobs=10, verbose=True ), ) if regress_type not in { 'cmf' }: regressor.fit(X_obs, y_obs) #print(regressor.model_.kernel_.get_params()) # Debug. kwargs['regressor'] = regressor return kwargs
def learn_success_model(self, action_parameters: np.ndarray, labels: np.ndarray) -> None: kernel = C(1., (0.1, 100)) * RBF(1., (1e-2, 1e2)) gpr = GaussianProcessRegressor(kernel=kernel).fit( action_parameters, labels) self.gpr = gpr
import numpy as np import pandas as pd from sklearn.model_selection import ShuffleSplit, train_test_split from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.gaussian_process.kernels import WhiteKernel as W, ConstantKernel as C, Matern, RBF, ExpSineSquared, RationalQuadratic from sklearn.metrics import r2_score np.random.seed(0) data = pd.read_csv("/home/lguo1/Desktop/train.csv") # print('data\n%s'%(data.head())) prices = data['medv'] features = data[['rm', 'lstat', 'ptratio', 'black']] X_train, X_test, y_train, y_test = train_test_split(features, prices, test_size=1 / 3) #kernel = C(10, (1e-3, 1e3)) + C(1, (1e-3, 1e3)) * Matern(length_scale=1.0, length_scale_bounds=(1e-2, 1e2), nu=1.5) + W() #kernel = C(10, (1e-3, 1e3)) + C(1, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2)) + W() #kernel = C(10, (1e-3, 1e3)) + C(1, (1e-3, 1e3)) * ExpSineSquared(length_scale=1.0, periodicity=3.0, length_scale_bounds=(0.1, 10.0), periodicity_bounds=(1.0, 10.0)) + W() #kernel = C(10, (1e-3, 1e3)) + C(1, (1e-3, 1e3)) * RationalQuadratic(length_scale=1.0, alpha=0.1) + W() kernel = W() + C(10, (1e-3, 1e3)) + C(1, (1e-3, 1e3)) * RBF( 10, (1e-2, 1e2)) + C(1, (1e-3, 1e3)) * Matern( length_scale=1.0, length_scale_bounds=(1e-2, 1e2), nu=1.5) #kernel = W() + C(10, (1e-3, 1e3)) + RBF(10, (1e-3, 1e3)) + Matern(length_scale=1.0, length_scale_bounds=(1e-3, 1e3), nu=1.5) gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9) gp.fit(X_train, y_train) print("Posterior (kernel: %s)\n Log-Likelihood: %.3f" % (gp.kernel_, gp.log_marginal_likelihood(gp.kernel_.theta))) print('training result: %f' % r2_score(y_train, gp.predict(X_train))) print('test result: %f' % r2_score(y_test, gp.predict(X_test)))
import pytest def f(x): return x * np.sin(x) X = np.atleast_2d([1., 3., 5., 6., 7., 8.]).T X2 = np.atleast_2d([2., 4., 5.5, 6.5, 7.5]).T y = f(X).ravel() fixed_kernel = RBF(length_scale=1.0, length_scale_bounds="fixed") kernels = [RBF(length_scale=1.0), fixed_kernel, RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)), C(1.0, (1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)), C(1.0, (1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)) + C(1e-5, (1e-5, 1e2)), C(0.1, (1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)) + WhiteKernel(1e-2, (1e-5, 1e2)) ] non_fixed_kernels = [kernel for kernel in kernels if kernel != fixed_kernel] @pytest.mark.parametrize('kernel', kernels) def test_gpr_interpolation(kernel): # Test the interpolating property for different kernels.
assert_almost_equal, assert_equal, assert_raise_message) def f(x): return x * np.sin(x) X = np.atleast_2d([1., 3., 5., 6., 7., 8.]).T X2 = np.atleast_2d([2., 4., 5.5, 6.5, 7.5]).T y = f(X).ravel() fixed_kernel = RBF(length_scale=1.0, length_scale_bounds="fixed") kernels = [ RBF(length_scale=1.0), fixed_kernel, RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)), C(1.0, (1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)), C(1.0, (1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)) + C(1e-5, (1e-5, 1e2)), C(0.1, (1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)) + C(1e-5, (1e-5, 1e2)) ] def test_gpr_interpolation(): # Test the interpolating property for different kernels. for kernel in kernels: gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) y_pred, y_cov = gpr.predict(X, return_cov=True)
from sklearn.utils.testing \ import (assert_true, assert_greater, assert_array_less, assert_almost_equal, assert_equal) def f(x): return x * np.sin(x) X = np.atleast_2d([1., 3., 5., 6., 7., 8.]).T X2 = np.atleast_2d([2., 4., 5.5, 6.5, 7.5]).T y = f(X).ravel() fixed_kernel = RBF(length_scale=1.0, length_scale_bounds="fixed") kernels = [RBF(length_scale=1.0), fixed_kernel, RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)), C(1.0, (1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)), C(1.0, (1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)) + C(1e-5, (1e-5, 1e2)), C(0.1, (1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)) + C(1e-5, (1e-5, 1e2))] def test_gpr_interpolation(): """Test the interpolating property for different kernels.""" for kernel in kernels: gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) y_pred, y_cov = gpr.predict(X, return_cov=True)
def regress_variable_on_pseudotime(pseudotime, vals, TrajName, var_name, var_type, producePlot=True, verbose=False, Continuous_Regression_Type='linear', R2_Threshold=0.5, max_sample=-1, alpha_factor=2): """ Auxillary function for regression_of_variable_with_trajectories() Continuous_Regression_Type can be 'linear','gpr' for Gaussian Process, 'kr' for kernel ridge """ if var_type == 'BINARY': #convert back to binary vals mn = min(vals) mx = max(vals) vals[np.where(vals == mn)] = 0 vals[np.where(vals == mx)] = 1 if len(np.unique(vals)) == 1: regressor = None else: regressor = LogisticRegression(random_state=0, max_iter=1000, penalty='none').fit( pseudotime, vals) if var_type == 'CATEGORICAL': if len(np.unique(vals)) == 1: regressor = None else: regressor = LogisticRegression(random_state=0, max_iter=1000, penalty='none').fit( pseudotime, vals) if var_type == 'CONTINUOUS' or var_type == 'ORDINAL': if len(np.unique(vals)) == 1: regressor = None else: if Continuous_Regression_Type == 'gpr': # subsampling if needed pst = pseudotime.copy() vls = vals.copy() if max_sample > 0: l = list(range(len(vals))) random.shuffle(l) index_value = random.sample(l, min(max_sample, len(vls))) pst = pst[index_value] vls = vls[index_value] if len(np.unique(vls)) > 1: gp_kernel = C(1.0, (1e-3, 1e3)) * RBF(1, (1e-2, 1e2)) #gp_kernel = RBF(np.std(vals)) regressor = GaussianProcessRegressor(kernel=gp_kernel, alpha=np.var(vls) * alpha_factor) regressor.fit(pst, vls) else: regressor = None if Continuous_Regression_Type == 'linear': regressor = LinearRegression() regressor.fit(pseudotime, vals) r2score = 0 if regressor is not None: r2score = r2_score(vals, regressor.predict(pseudotime)) if producePlot and r2score > R2_Threshold: plt.plot(pseudotime, vals, 'ro', label='data') unif_pst = np.linspace(min(pseudotime), max(pseudotime), 100) pred = regressor.predict(unif_pst) if var_type == 'BINARY' or var_type == 'CATEGORICAL': prob = regressor.predict_proba(unif_pst) plt.plot(unif_pst, prob[:, 1], 'g-', linewidth=2, label='proba') if var_type == 'CONTINUOUS' or var_type == 'ORDINAL': plt.plot(unif_pst, pred, 'g-', linewidth=2, label='predicted') bincenters, wav = moving_weighted_average(pseudotime, vals.reshape(-1, 1), step_size=1.5) plt.plot(bincenters, fill_gaps_in_number_sequence(wav), 'b-', linewidth=2, label='sliding av') plt.xlabel('Pseudotime', fontsize=20) plt.ylabel(var_name, fontsize=20) plt.title(TrajName + ', r2={:2.2f}'.format(r2score), fontsize=20) plt.legend(fontsize=15) plt.show() return r2score, regressor
def predict(self, X, return_std=False, return_cov=False): """Predict using the Gaussian process regression model We can also predict based on an unfitted model by using the GP prior. In addition to the mean of the predictive distribution, also its standard deviation (return_std=True) or covariance (return_cov=True). Note that at most one of the two can be requested. Parameters ---------- X : array-like, shape = (n_samples, n_features) Query points where the GP is evaluated return_std : bool, default: False If True, the standard-deviation of the predictive distribution at the query points is returned along with the mean. return_cov : bool, default: False If True, the covariance of the joint predictive distribution at the query points is returned along with the mean Returns ------- y_mean : array, shape = (n_samples, [n_output_dims]) Mean of predictive distribution a query points y_std : array, shape = (n_samples,), optional Standard deviation of predictive distribution at query points. Only returned when return_std is True. y_cov : array, shape = (n_samples, n_samples), optional Covariance of joint predictive distribution a query points. Only returned when return_cov is True. """ if return_std and return_cov: raise RuntimeError( "Not returning standard deviation of predictions when " "returning full covariance.") X = check_array(X) if not hasattr(self, "X_train_"): # Unfitted;predict based on GP prior if self.kernel is None: kernel = (C(1.0, constant_value_bounds="fixed") * RBF(1.0, length_scale_bounds="fixed")) else: kernel = self.kernel y_mean = np.zeros(X.shape[0]) if return_cov: y_cov = kernel(X) return y_mean, y_cov elif return_std: y_var = kernel.diag(X) return y_mean, np.sqrt(y_var) else: return y_mean else: # Predict based on GP posterior K_trans = self.kernel_(X, self.X_train_) y_mean = K_trans.dot(self.alpha_) # Line 4 (y_mean = f_star) y_mean = self._y_train_mean + y_mean # undo normal. if return_cov: v = cho_solve((self.L_, True), K_trans.T) # Line 5 y_cov = self.kernel_(X) - K_trans.dot(v) # Line 6 return y_mean, y_cov elif return_std: # cache result of K_inv computation if self._K_inv is None: # compute inverse K_inv of K based on its Cholesky # decomposition L and its inverse L_inv L_inv = solve_triangular(self.L_.T, np.eye(self.L_.shape[0])) self._K_inv = L_inv.dot(L_inv.T) # Compute variance of predictive distribution y_var = self.kernel_.diag(X) y_var -= np.einsum("ij,ij->i", np.dot(K_trans, self._K_inv), K_trans) # Check if any of the variances is negative because of # numerical issues. If yes: set the variance to 0. y_var_negative = y_var < 0 if np.any(y_var_negative): warnings.warn("Predicted variances smaller than 0. " "Setting those variances to 0.") y_var[y_var_negative] = 0.0 return y_mean, np.sqrt(y_var) else: return y_mean
bkghist_template.Rebin(8) bkghist_template.Scale(stats/bkghist_template.Integral()) trainHisto = bkghist_template.Clone('trainHisto') optKernel = None GPh = GPHisto(bkghist_template) X = GPh.getXArr() y = GPh.getYArr() dy = GPh.getErrArr() x = GPh.getXArr() kernel = C(10.0, (1e-3, 1e15)) * RBF(50.0, (1e-2, 1e5)) #squared exponential kernel #kernel = C(1000.0, (1e-3, 1e15)) * FallExp(1.0, (1e-5, 1e2), 1.0, (1e-3,1e15)) * Gibbs(0.01, (1e-7, 1e5), 0.01, (1e-7,1e5)) gp = GaussianProcessRegressor(kernel=kernel #,optimizer='fmin' ,alpha=dy**2 ,n_restarts_optimizer=15 ) gp.fit(X,y) print gp.kernel_ lengthOpt = float(re.search('length_scale=(\d+(\.\d+)?)', gp.kernel_.__repr__()).group(1)) #raise() optKernel = gp.kernel_
X = np.atleast_2d(np.linspace(0, 10, 30)).T X2 = np.atleast_2d([2., 4., 5.5, 6.5, 7.5]).T y = np.array(f(X).ravel() > 0, dtype=int) fX = f(X).ravel() y_mc = np.empty(y.shape, dtype=int) # multi-class y_mc[fX < -0.35] = 0 y_mc[(fX >= -0.35) & (fX < 0.35)] = 1 y_mc[fX > 0.35] = 2 fixed_kernel = RBF(length_scale=1.0, length_scale_bounds="fixed") kernels = [ RBF(length_scale=0.1), fixed_kernel, RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)), C(1.0, (1e-2, 1e2)) * RBF(length_scale=1.0, length_scale_bounds=(1e-3, 1e3)) ] non_fixed_kernels = [kernel for kernel in kernels if kernel != fixed_kernel] @pytest.mark.parametrize('kernel', kernels) def test_predict_consistent(kernel): # Check binary predict decision has also predicted probability above 0.5. gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y) assert_array_equal(gpc.predict(X), gpc.predict_proba(X)[:, 1] >= 0.5) def test_predict_consistent_structured(): # Check binary predict decision has also predicted probability above 0.5. X = ['A', 'AB', 'B'] y = np.array([True, False, True])
#######Special Credits for developing the code : AV Sir's lecture material ########### import pandas as pd import numpy as np from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C from matplotlib import pyplot as plt data = [[0, -45], [2, -58], [4, -36], [6, -59], [8, -36], [10, -55], [11, -64]] Train = pd.DataFrame(data, columns=['Distance', 'Strength']) data1 = [[1, -51], [3, -63], [5, -52], [7, -62], [9, -43]] Test = pd.DataFrame(data1, columns=['Distance', 'Strength']) kernel = C(1.0, (1e-10, 1e10)) * RBF(2, (1e-4, 1e4)) gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=10, random_state=2) #print(gp) gp.fit((np.array(Train["Distance"])).reshape(-1, 1), Train["Strength"]) y_pred, sigma = gp.predict((np.array(Test["Distance"])).reshape(-1, 1), return_std=True) for i in range(0, len(y_pred)): print("Distance", Test["Distance"][i]) print("Actual value of Signal Strength", Test["Strength"][i]) print("predicted mean of Signal Strength", y_pred[i]) print("variance", sigma[i]) print("\n") #print(Test["Strength"])