def __init__(self, X, y, Xtest, kernel='Matern52', lengthscale=None, lengthscale_init=None, iterations=50, learning_rate=.1, grid_points_ratio=1., maxroot=100, num_batches=10, calculate_sd=0, use_gpu=1, verbose=0, seed=0): """ Initiates reconstructor parameters and pre-processes training and test data arrays """ torch.manual_seed(seed) input_dim = np.ndim(y) X, y = gprutils.prepare_training_data(X, y) Xtest = gprutils.prepare_test_data(Xtest) self.X, self.y, self.Xtest = X, y, Xtest self.toeplitz = gpytorch.settings.use_toeplitz(True) self.maxroot = gpytorch.settings.max_root_decomposition_size(maxroot) if use_gpu and torch.cuda.is_available(): torch.cuda.empty_cache() torch.cuda.manual_seed_all(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.set_default_tensor_type(torch.cuda.DoubleTensor) self.X, self.y = self.X.cuda(), self.y.cuda() self.Xtest = self.Xtest.cuda() self.toeplitz = gpytorch.settings.use_toeplitz(False) else: torch.set_default_tensor_type(torch.DoubleTensor) self.likelihood = gpytorch.likelihoods.GaussianLikelihood() _kernel = get_kernel(kernel, input_dim, use_gpu, lengthscale=lengthscale, lengthscale_init=lengthscale_init) self.model = skgprmodel(self.X, self.y, _kernel, self.likelihood, input_dim, grid_points_ratio) if use_gpu: self.model.cuda() self.iterations = iterations self.num_batches = num_batches self.calculate_sd = calculate_sd self.lr = learning_rate self.lscales, self.noise_all = [], [] self.hyperparams = { "lengthscale": self.lscales, "noise": self.noise_all, } self.verbose = verbose
def predict(self, Xtest=None, **kwargs): """ Uses trained GP regression model to make predictions Args: Xtest (ndarray): "Test" points (for prediction with a trained GP model) with dimensions :math:`N \\times M` or :math:`N \\times M \\times L`. Uses Xtest from __init__ by default. If Xtest is None, uses training data X. Returns: Predictive mean and standard deviation """ if Xtest is None and self.Xtest is None: warnings.warn( "No test data provided. Using training data for prediction", UserWarning) self.Xtest = self.X elif Xtest is not None: self.Xtest = gprutils.prepare_test_data(Xtest, precision=self.precision) self.fulldims = Xtest.shape[1:] if next(self.model.parameters()).is_cuda: self.Xtest = self.Xtest.cuda() if kwargs.get("verbose") is not None: self.verbose = kwargs.get("verbose") if self.verbose: print("Calculating predictive mean and variance...", end=" ") with torch.no_grad(): mean, cov = self.model(self.Xtest, full_cov=False, noiseless=False) mean = mean.cpu().numpy() sd = cov.sqrt().cpu().numpy() mean = mean.reshape(self.fulldims) sd = sd.reshape(self.fulldims) if self.verbose: print("Done") return mean, sd
def predict(self, **kwargs): """ Makes a prediction with trained GP regression model """ if kwargs.get("Xtest") is not None: self.Xtest = gprutils.prepare_test_data(kwargs.get("Xtest")) if next(self.model.parameters()).is_cuda: self.Xtest = self.Xtest.cuda() if kwargs.get("num_batches") is not None: self.num_batches = kwargs.get("num_batches") if kwargs.get("max_root") is not None: self.max_root = kwargs.get("max_root") self.model.eval() self.likelihood.eval() batch_range = len(self.Xtest) // self.num_batches mean = np.zeros((self.Xtest.shape[0])) if self.calculate_sd: sd = np.zeros((self.Xtest.shape[0])) if self.calculate_sd: print('Calculating predictive mean and uncertainty...') else: print('Calculating predictive mean...') for i in range(self.num_batches): print("\rBatch {}/{}".format(i + 1, self.num_batches), end="") Xtest_i = self.Xtest[i * batch_range:(i + 1) * batch_range] with torch.no_grad(), gpytorch.settings.fast_pred_var( ), self.toeplitz, self.maxroot: covar_i = self.likelihood(self.model(Xtest_i)) mean[i * batch_range:(i + 1) * batch_range] = covar_i.mean.cpu().numpy() if self.calculate_sd: sd[i * batch_range:(i + 1) * batch_range] = covar_i.stddev.cpu().numpy() print("\nDone") if self.calculate_sd: return (mean, sd) return mean
def __init__(self, X, y, Xtest=None, kernel='RBF', lengthscale=None, ski=True, learning_rate=.1, iterations=50, use_gpu=1, verbose=1, seed=0, **kwargs): """ Initiates reconstructor parameters and pre-processes training and test data arrays """ self.precision = kwargs.get("precision", "double") if self.precision == 'single': self.tensor_type = torch.FloatTensor self.tensor_type_gpu = torch.cuda.FloatTensor else: self.tensor_type = torch.DoubleTensor self.tensor_type_gpu = torch.cuda.DoubleTensor torch.manual_seed(seed) if use_gpu and torch.cuda.is_available(): torch.cuda.empty_cache() torch.cuda.manual_seed_all(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.set_default_tensor_type(self.tensor_type_gpu) input_dim = np.ndim(y) if Xtest is not None: self.fulldims = Xtest.shape[1:] else: self.fulldims = X.shape[1:] X, y = gprutils.prepare_training_data(X, y, precision=self.precision) if Xtest is not None: Xtest = gprutils.prepare_test_data(Xtest, precision=self.precision) self.X, self.y, self.Xtest = X, y, Xtest self.do_ski = ski if kernel == "Spectral": self.do_ski = False self.toeplitz = gpytorch.settings.use_toeplitz(True) maxroot = kwargs.get("maxroot", 100) self.maxroot = gpytorch.settings.max_root_decomposition_size(maxroot) if use_gpu and torch.cuda.is_available(): self.X, self.y = self.X.cuda(), self.y.cuda() if self.Xtest is not None: self.Xtest = self.Xtest.cuda() self.toeplitz = gpytorch.settings.use_toeplitz(False) else: torch.set_default_tensor_type(self.tensor_type) self.likelihood = gpytorch.likelihoods.GaussianLikelihood() isotropic = kwargs.get("isotropic") n_mixtures = kwargs.get("n_mixtures") _kernel = gpytorch_kernels.get_kernel(kernel, input_dim, use_gpu, lengthscale=lengthscale, isotropic=isotropic, precision=self.precision, n_mixtures=n_mixtures) grid_points_ratio = kwargs.get("grid_points_ratio", 1.) self.model = skgprmodel(self.X, self.y, _kernel, self.likelihood, input_dim, grid_points_ratio, self.do_ski) if use_gpu: self.model.cuda() self.iterations = iterations self.num_batches = kwargs.get("num_batches", 1) self.learning_rate = learning_rate self.noise_all = [] if kernel == "Spectral": self.scales, self.means, self.weights = [], [], [] self.hyperparams = { "scales": self.scales, "means": self.means, "weights": self.weights, "noise": self.noise_all, "maxdim": max(self.fulldims) } else: self.lscales = [] self.hyperparams = { "lengthscale": self.lscales, "noise": self.noise_all, } self.verbose = verbose
def predict(self, Xtest=None, **kwargs): """ Makes a prediction with trained GP regression model Args: Xtest (ndarray): "Test" points (for prediction with a trained GP model) with dimension :math:`N \\times M`, :math:`N \\times M \\times L` or :math:`N \\times M \\times L \\times K` max_root (int): Maximum number of Lanczos iterations to perform in prediction stage num_batches (int): Number of batches for splitting the Xtest array (for large datasets, you may not have enough GPU memory to process the entire dataset at once) """ def predict_(Xtest_i): with torch.no_grad(), gpytorch.settings.fast_pred_var( ), self.toeplitz, self.maxroot: covar_i = self.likelihood(self.model(Xtest_i)) mean_i, sd_i = covar_i.mean, covar_i.stddev return mean_i, sd_i if Xtest is None and self.Xtest is None: warnings.warn( "No test data provided. Using training data for prediction", UserWarning) self.Xtest = self.X elif Xtest is not None: self.Xtest = gprutils.prepare_test_data(Xtest, precision=self.precision) self.fulldims = Xtest.shape[1:] if next(self.model.parameters()).is_cuda: self.Xtest = self.Xtest.cuda() if kwargs.get("verbose") is not None: self.verbose = kwargs.get("verbose") if kwargs.get("num_batches") is not None: self.num_batches = kwargs.get("num_batches") if kwargs.get("max_root") is not None: self.max_root = kwargs.get("max_root") self.model.eval() self.likelihood.eval() batch_size = len(self.Xtest) // self.num_batches dtype_ = np.float32 if self.precision == 'single' else np.float64 mean = np.zeros((self.Xtest.shape[0]), dtype_) sd = np.zeros((self.Xtest.shape[0]), dtype_) if self.verbose: print('Calculating predictive mean and uncertainty...') for i in range(self.num_batches): if self.verbose: print("\rBatch {}/{}".format(i + 1, self.num_batches), end="") Xtest_i = self.Xtest[i * batch_size:(i + 1) * batch_size] mean_i, sd_i = predict_(Xtest_i) mean[i * batch_size:(i + 1) * batch_size] = mean_i.cpu().numpy() sd[i * batch_size:(i + 1) * batch_size] = sd_i.cpu().numpy() Xtest_i = self.Xtest[(i + 1) * batch_size:] if len(Xtest_i) > 0: mean_i, sd_i = predict_(Xtest_i) mean[(i + 1) * batch_size:] = mean_i.cpu().numpy() sd[(i + 1) * batch_size:] = sd_i.cpu().numpy() sd = sd.reshape(self.fulldims) mean = mean.reshape(self.fulldims) if self.verbose: print("\nDone") return mean, sd
def __init__(self, X, y, Xtest=None, kernel='RBF', lengthscale=None, sparse=False, indpoints=None, learning_rate=5e-2, iterations=1000, use_gpu=False, verbose=1, seed=0, **kwargs): """ Initiates reconstructor parameters and pre-processes training and test data arrays """ self.precision = kwargs.get("precision", "double") if self.precision == 'single': self.tensor_type = torch.FloatTensor self.tensor_type_gpu = torch.cuda.FloatTensor else: self.tensor_type = torch.DoubleTensor self.tensor_type_gpu = torch.cuda.DoubleTensor self.verbose = verbose torch.manual_seed(seed) pyro.set_rng_seed(seed) pyro.clear_param_store() if use_gpu and torch.cuda.is_available(): torch.cuda.empty_cache() torch.cuda.manual_seed_all(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.set_default_tensor_type(self.tensor_type_gpu) use_gpu = True else: torch.set_default_tensor_type(self.tensor_type) use_gpu = False input_dim = np.ndim(y) self.X, self.y = gprutils.prepare_training_data( X, y, precision=self.precision) self.do_sparse = sparse if lengthscale is None and not kwargs.get("isotropic"): lengthscale = [[0. for l in range(input_dim)], [np.mean(y.shape) / 2 for l in range(input_dim)] ] # TODO Make separate lscale for each dim elif lengthscale is None and kwargs.get("isotropic"): lengthscale = [0., np.mean(y.shape) / 2] kernel = pyro_kernels.get_kernel(kernel, input_dim, lengthscale, use_gpu, amplitude=kwargs.get('amplitude'), precision=self.precision) if Xtest is not None: self.fulldims = Xtest.shape[1:] else: self.fulldims = X.shape[1:] if Xtest is not None: self.Xtest = gprutils.prepare_test_data(Xtest, precision=self.precision) else: self.Xtest = Xtest if use_gpu: self.X = self.X.cuda() self.y = self.y.cuda() if self.Xtest is not None: self.Xtest = self.Xtest.cuda() if not self.do_sparse: self.model = gp.models.GPRegression(self.X, self.y, kernel) else: if indpoints is None: indpoints = len(self.X) // 10 indpoints = indpoints + 1 if indpoints == 0 else indpoints else: indpoints = len( self.X) if indpoints > len(self.X) else indpoints Xu = self.X[::len(self.X) // indpoints] if self.verbose == 2: print( "# of inducing points for sparse GP regression: {}".format( len(Xu))) self.model = gp.models.SparseGPRegression(self.X, self.y, kernel, Xu, jitter=1.0e-5) if use_gpu: self.model.cuda() self.learning_rate = learning_rate self.iterations = iterations self.hyperparams = {} self.indpoints_all = [] self.lscales, self.noise_all, self.amp_all = [], [], [] self.hyperparams = { "lengthscale": self.lscales, "noise": self.noise_all, "variance": self.amp_all, "inducing_points": self.indpoints_all }
def __init__(self, X, y, Xtest=None, kernel='RBF', lengthscale=None, independent=False, learning_rate=.1, iterations=50, use_gpu=1, verbose=1, seed=0, **kwargs): """ Initiates reconstructor parameters and pre-processes training and test data arrays """ self.precision = kwargs.get("precision", "double") if self.precision == 'single': self.tensor_type = torch.FloatTensor self.tensor_type_gpu = torch.cuda.FloatTensor else: self.tensor_type = torch.DoubleTensor self.tensor_type_gpu = torch.cuda.DoubleTensor torch.manual_seed(seed) if use_gpu and torch.cuda.is_available(): torch.cuda.empty_cache() torch.cuda.manual_seed_all(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.set_default_tensor_type(self.tensor_type_gpu) input_dim = np.ndim(y) - 1 X, y = gprutils.prepare_training_data(X, y, vector_valued=True, precision=self.precision) num_tasks = y.shape[-1] if Xtest is not None: self.fulldims = Xtest.shape[1:] + (num_tasks, ) else: self.fulldims = X.shape[1:] + (num_tasks, ) if Xtest is not None: Xtest = gprutils.prepare_test_data(Xtest, precision=self.precision) self.X, self.y, self.Xtest = X, y, Xtest self.toeplitz = gpytorch.settings.use_toeplitz(True) maxroot = kwargs.get("maxroot", 100) self.maxroot = gpytorch.settings.max_root_decomposition_size(maxroot) if use_gpu and torch.cuda.is_available(): self.X, self.y = self.X.cuda(), self.y.cuda() if self.Xtest is not None: self.Xtest = self.Xtest.cuda() self.toeplitz = gpytorch.settings.use_toeplitz(False) else: torch.set_default_tensor_type(self.tensor_type) self.likelihood = gpytorch.likelihoods.MultitaskGaussianLikelihood( num_tasks) isotropic = kwargs.get("isotropic") _kernel = gpytorch_kernels.get_kernel(kernel, input_dim, use_gpu, lengthscale=lengthscale, isotropic=isotropic, precision=self.precision) if not independent: self.model = vgprmodel(self.X, self.y, _kernel, self.likelihood, num_tasks) else: self.model = ivgprmodel(self.X, self.y, _kernel, self.likelihood, num_tasks) if use_gpu: self.model.cuda() self.iterations = iterations self.num_batches = kwargs.get("num_batches", 1) self.learning_rate = learning_rate self.independent = independent self.lscales = [] self.hyperparams = { "lengthscale": self.lscales, # need to add noise as well } self.verbose = verbose
def __init__(self, X, y, Xtest, kernel, lengthscale=None, indpoints=1000, learning_rate=5e-2, iterations=1000, use_gpu=False, verbose=False, seed=0, **kwargs): """ Initiates reconstructor parameters and pre-processes training and test data arrays """ torch.manual_seed(seed) pyro.set_rng_seed(seed) pyro.clear_param_store() if use_gpu and torch.cuda.is_available(): torch.cuda.empty_cache() torch.cuda.manual_seed_all(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.set_default_tensor_type(torch.cuda.DoubleTensor) use_gpu = True else: torch.set_default_tensor_type(torch.DoubleTensor) use_gpu = False input_dim = np.ndim(y) self.X, self.y = gprutils.prepare_training_data(X, y) if indpoints > len(self.X): indpoints = len(self.X) Xu = self.X[::len(self.X) // indpoints] if lengthscale is None: lengthscale = [[0. for l in range(input_dim)], [np.mean(y.shape) / 2 for l in range(input_dim)]] kernel = get_kernel(kernel, input_dim, lengthscale, use_gpu, amplitude=kwargs.get('amplitude')) self.fulldims = Xtest.shape[1:] self.Xtest = gprutils.prepare_test_data(Xtest) if use_gpu: self.X = self.X.cuda() self.y = self.y.cuda() self.Xtest = self.Xtest.cuda() self.sgpr = gp.models.SparseGPRegression( self.X, self.y, kernel, Xu, jitter=1.0e-5) print("# of inducing points for GP regression: {}".format(len(Xu))) if use_gpu: self.sgpr.cuda() self.learning_rate = learning_rate self.iterations = iterations self.hyperparams = {} self.indpoints_all = [] self.lscales, self.noise_all, self.amp_all = [], [], [] self.hyperparams = { "lengthscale": self.lscales, "noise": self.noise_all, "variance": self.amp_all, "inducing_points": self.indpoints_all } self.verbose = verbose