def _get_fixed_noise_model_single_output(**tkwargs): train_X, train_Y = _get_random_mt_data(**tkwargs) train_Yvar = torch.full_like(train_Y, 0.05) model = FixedNoiseMultiTaskGP( train_X, train_Y, train_Yvar, task_feature=1, output_tasks=[1] ) return model.to(**tkwargs)
def _get_fixed_noise_model_single_output(**tkwargs): train_X, train_Y = _get_random_mt_data(**tkwargs) train_Yvar = torch.full_like(train_Y, 0.05) model = FixedNoiseMultiTaskGP( train_X, train_Y, train_Yvar, task_feature=1, output_tasks=[1] ) return model.to(**tkwargs)
def _get_fixed_noise_model_and_training_data(input_transform=None, **tkwargs): train_X, train_Y = _get_random_mt_data(**tkwargs) train_Yvar = torch.full_like(train_Y, 0.05) model = FixedNoiseMultiTaskGP( train_X, train_Y, train_Yvar, task_feature=1, input_transform=input_transform ) return model.to(**tkwargs), train_X, train_Y, train_Yvar
def _get_fixed_noise_and_given_covar_module_model(**tkwargs): train_X, train_Y = _get_random_mt_data(**tkwargs) train_Yvar = torch.full_like(train_Y, 0.05) model = FixedNoiseMultiTaskGP( train_X, train_Y, train_Yvar, task_feature=1, covar_module=MaternKernel(nu=1.5, lengthscale_prior=GammaPrior(1.0, 1.0)), ) return model.to(**tkwargs)
def _get_fixed_noise_and_prior_model(**tkwargs): train_X, train_Y = _get_random_mt_data(**tkwargs) train_Yvar = torch.full_like(train_Y, 0.05) sd_prior = GammaPrior(2.0, 0.15) sd_prior._event_shape = torch.Size([2]) model = FixedNoiseMultiTaskGP( train_X, train_Y, train_Yvar, task_feature=1, task_covar_prior=LKJCovariancePrior(2, 0.6, sd_prior), ) return model.to(**tkwargs)
def _get_model( X: Tensor, Y: Tensor, Yvar: Tensor, task_feature: Optional[int] = None, fidelity_features: Optional[List[int]] = None, **kwargs: Any, ) -> GPyTorchModel: """Instantiate a model of type depending on the input data. Args: X: A `n x d` tensor of input features. Y: A `n x m` tensor of input observations. Yvar: A `n x m` tensor of input variances (NaN if unobserved). task_feature: The index of the column pertaining to the task feature (if present). fidelity_features: List of columns of X that are fidelity parameters. Returns: A GPyTorchModel (unfitted). """ Yvar = Yvar.clamp_min_(MIN_OBSERVED_NOISE_LEVEL) is_nan = torch.isnan(Yvar) any_nan_Yvar = torch.any(is_nan) all_nan_Yvar = torch.all(is_nan) if any_nan_Yvar and not all_nan_Yvar: raise ValueError( "Mix of known and unknown variances indicates valuation function " "errors. Variances should all be specified, or none should be." ) if fidelity_features is None: fidelity_features = [] if len(fidelity_features) == 0: # only pass linear_truncated arg if there are fidelities kwargs = {k: v for k, v in kwargs.items() if k != "linear_truncated"} if len(fidelity_features) > 0: if task_feature: raise NotImplementedError( "multi-task multi-fidelity models not yet available" ) # at this point we can assume that there is only a single fidelity parameter gp = SingleTaskMultiFidelityGP( train_X=X, train_Y=Y, data_fidelity=fidelity_features[0], **kwargs ) elif task_feature is None and all_nan_Yvar: gp = SingleTaskGP(train_X=X, train_Y=Y, **kwargs) elif task_feature is None: gp = FixedNoiseGP(train_X=X, train_Y=Y, train_Yvar=Yvar, **kwargs) elif all_nan_Yvar: gp = MultiTaskGP(train_X=X, train_Y=Y, task_feature=task_feature, **kwargs) else: gp = FixedNoiseMultiTaskGP( train_X=X, train_Y=Y.view(-1), train_Yvar=Yvar.view(-1), task_feature=task_feature, **kwargs, ) return gp
def _get_model(X: Tensor, Y: Tensor, Yvar: Tensor, task_feature: Optional[int]) -> GPyTorchModel: """Instantiate a model of type depending on the input data.""" Yvar = Yvar.clamp_min_(MIN_OBSERVED_NOISE_LEVEL) # pyre-ignore: [16] if task_feature is None: gp = FixedNoiseGP(train_X=X, train_Y=Y, train_Yvar=Yvar) else: gp = FixedNoiseMultiTaskGP( train_X=X, train_Y=Y.view(-1), train_Yvar=Yvar.view(-1), task_feature=task_feature, ) return gp
def _get_model( X: Tensor, Y: Tensor, Yvar: Tensor, task_feature: Optional[int], fidelity_features: Optional[List[int]] = None, fidelity_model_id: Optional[int] = None, ) -> GPyTorchModel: """Instantiate a model of type depending on the input data.""" Yvar = Yvar.clamp_min_(MIN_OBSERVED_NOISE_LEVEL) # pyre-ignore: [16] is_nan = torch.isnan(Yvar) any_nan_Yvar = torch.any(is_nan) all_nan_Yvar = torch.all(is_nan) if any_nan_Yvar and not all_nan_Yvar: raise ValueError( "Mix of known and unknown variances indicates " "valuation function errors. Variances should all be specified, or " "none should be.") if fidelity_features is None: fidelity_features = [] if fidelity_model_id is None or len(fidelity_features) == 0: if task_feature is None and all_nan_Yvar: gp = SingleTaskGP(train_X=X, train_Y=Y) elif task_feature is None: gp = FixedNoiseGP(train_X=X, train_Y=Y, train_Yvar=Yvar) elif all_nan_Yvar: gp = MultiTaskGP(train_X=X, train_Y=Y, task_feature=task_feature) else: gp = FixedNoiseMultiTaskGP( train_X=X, train_Y=Y.view(-1), train_Yvar=Yvar.view(-1), task_feature=task_feature, ) else: gp_model = model_list[fidelity_model_id] # pyre-ignore [29] gp = gp_model(train_X=X, train_Y=Y, train_data_fidelity=False) return gp
def test_FixedNoiseMultiTaskGP(self): bounds = torch.tensor([[-1.0, 0.0], [1.0, 1.0]]) for dtype, use_intf in itertools.product( (torch.float, torch.double), (False, True) ): tkwargs = {"device": self.device, "dtype": dtype} intf = ( Normalize( d=2, bounds=bounds.to(**tkwargs), transform_on_preprocess=True ) if use_intf else None ) model, train_X, _, _ = _get_fixed_noise_model_and_training_data( input_transform=intf, **tkwargs ) self.assertIsInstance(model, FixedNoiseMultiTaskGP) self.assertEqual(model.num_outputs, 2) self.assertIsInstance(model.likelihood, FixedNoiseGaussianLikelihood) self.assertIsInstance(model.mean_module, ConstantMean) self.assertIsInstance(model.covar_module, ScaleKernel) matern_kernel = model.covar_module.base_kernel self.assertIsInstance(matern_kernel, MaternKernel) self.assertIsInstance(matern_kernel.lengthscale_prior, GammaPrior) self.assertIsInstance(model.task_covar_module, IndexKernel) self.assertEqual(model._rank, 2) self.assertEqual( model.task_covar_module.covar_factor.shape[-1], model._rank ) if use_intf: self.assertIsInstance(model.input_transform, Normalize) # test model fitting mll = ExactMarginalLogLikelihood(model.likelihood, model) with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=OptimizationWarning) mll = fit_gpytorch_model(mll, options={"maxiter": 1}, max_retries=1) # check that training data has input transform applied # check that the train inputs have been transformed and set on the model if use_intf: self.assertTrue( torch.equal(model.train_inputs[0], model.input_transform(train_X)) ) # test posterior test_x = torch.rand(2, 1, **tkwargs) posterior_f = model.posterior(test_x) self.assertIsInstance(posterior_f, GPyTorchPosterior) self.assertIsInstance(posterior_f.mvn, MultitaskMultivariateNormal) self.assertEqual(posterior_f.mean.shape, torch.Size([2, 2])) self.assertEqual(posterior_f.variance.shape, torch.Size([2, 2])) # test that posterior w/ observation noise raises appropriate error with self.assertRaises(NotImplementedError): model.posterior(test_x, observation_noise=True) with self.assertRaises(NotImplementedError): model.posterior(test_x, observation_noise=torch.rand(2, **tkwargs)) # test posterior w/ single output index posterior_f = model.posterior(test_x, output_indices=[0]) self.assertIsInstance(posterior_f, GPyTorchPosterior) self.assertIsInstance(posterior_f.mvn, MultivariateNormal) self.assertEqual(posterior_f.mean.shape, torch.Size([2, 1])) self.assertEqual(posterior_f.variance.shape, torch.Size([2, 1])) # test posterior w/ bad output index with self.assertRaises(ValueError): model.posterior(test_x, output_indices=[2]) # test posterior (batch eval) test_x = torch.rand(3, 2, 1, **tkwargs) posterior_f = model.posterior(test_x) self.assertIsInstance(posterior_f, GPyTorchPosterior) self.assertIsInstance(posterior_f.mvn, MultitaskMultivariateNormal) # test that unsupported batch shape MTGPs throw correct error with self.assertRaises(ValueError): FixedNoiseMultiTaskGP( torch.rand(2, 2, 2), torch.rand(2, 2, 1), torch.rand(2, 2, 1), 0 ) # test that bad feature index throws correct error train_X, train_Y = _get_random_mt_data(**tkwargs) train_Yvar = torch.full_like(train_Y, 0.05) with self.assertRaises(ValueError): FixedNoiseMultiTaskGP(train_X, train_Y, train_Yvar, 2) # test that bad output task throws correct error with self.assertRaises(RuntimeError): FixedNoiseMultiTaskGP(train_X, train_Y, train_Yvar, 0, output_tasks=[2])
def test_FixedNoiseMultiTaskGP(self): for dtype in (torch.float, torch.double): tkwargs = {"device": self.device, "dtype": dtype} model = _get_fixed_noise_model(**tkwargs) self.assertIsInstance(model, FixedNoiseMultiTaskGP) self.assertEqual(model.num_outputs, 2) self.assertIsInstance(model.likelihood, FixedNoiseGaussianLikelihood) self.assertIsInstance(model.mean_module, ConstantMean) self.assertIsInstance(model.covar_module, ScaleKernel) matern_kernel = model.covar_module.base_kernel self.assertIsInstance(matern_kernel, MaternKernel) self.assertIsInstance(matern_kernel.lengthscale_prior, GammaPrior) self.assertIsInstance(model.task_covar_module, IndexKernel) self.assertEqual(model._rank, 2) self.assertEqual(model.task_covar_module.covar_factor.shape[-1], model._rank) # test model fitting mll = ExactMarginalLogLikelihood(model.likelihood, model) with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=OptimizationWarning) mll = fit_gpytorch_model(mll, options={"maxiter": 1}, max_retries=1) # test posterior test_x = torch.rand(2, 1, **tkwargs) posterior_f = model.posterior(test_x) self.assertIsInstance(posterior_f, GPyTorchPosterior) self.assertIsInstance(posterior_f.mvn, MultitaskMultivariateNormal) self.assertEqual(posterior_f.mean.shape, torch.Size([2, 2])) self.assertEqual(posterior_f.variance.shape, torch.Size([2, 2])) # test that posterior w/ observation noise raises appropriate error with self.assertRaises(NotImplementedError): model.posterior(test_x, observation_noise=True) with self.assertRaises(NotImplementedError): model.posterior(test_x, observation_noise=torch.rand(2, **tkwargs)) # test posterior w/ single output index posterior_f = model.posterior(test_x, output_indices=[0]) self.assertIsInstance(posterior_f, GPyTorchPosterior) self.assertIsInstance(posterior_f.mvn, MultivariateNormal) self.assertEqual(posterior_f.mean.shape, torch.Size([2, 1])) self.assertEqual(posterior_f.variance.shape, torch.Size([2, 1])) # test posterior w/ bad output index with self.assertRaises(ValueError): model.posterior(test_x, output_indices=[2]) # test posterior (batch eval) test_x = torch.rand(3, 2, 1, **tkwargs) posterior_f = model.posterior(test_x) self.assertIsInstance(posterior_f, GPyTorchPosterior) self.assertIsInstance(posterior_f.mvn, MultitaskMultivariateNormal) # test that unsupported batch shape MTGPs throw correct error with self.assertRaises(ValueError): FixedNoiseMultiTaskGP(torch.rand(2, 2, 2), torch.rand(2, 1), torch.rand(2, 1), 0) # test that bad feature index throws correct error train_X, train_Y = _get_random_mt_data(**tkwargs) train_Yvar = torch.full_like(train_Y, 0.05) with self.assertRaises(ValueError): FixedNoiseMultiTaskGP(train_X, train_Y, train_Yvar, 2) # test that bad output task throws correct error with self.assertRaises(RuntimeError): FixedNoiseMultiTaskGP(train_X, train_Y, train_Yvar, 0, output_tasks=[2])
def _get_model( X: Tensor, Y: Tensor, Yvar: Tensor, task_feature: Optional[int] = None, fidelity_features: Optional[List[int]] = None, use_input_warping: bool = False, **kwargs: Any, ) -> GPyTorchModel: """Instantiate a model of type depending on the input data. Args: X: A `n x d` tensor of input features. Y: A `n x m` tensor of input observations. Yvar: A `n x m` tensor of input variances (NaN if unobserved). task_feature: The index of the column pertaining to the task feature (if present). fidelity_features: List of columns of X that are fidelity parameters. Returns: A GPyTorchModel (unfitted). """ Yvar = Yvar.clamp_min(MIN_OBSERVED_NOISE_LEVEL) # pyre-ignore[16] is_nan = torch.isnan(Yvar) any_nan_Yvar = torch.any(is_nan) all_nan_Yvar = torch.all(is_nan) if any_nan_Yvar and not all_nan_Yvar: if task_feature: # TODO (jej): Replace with inferred noise before making perf judgements. Yvar[Yvar != Yvar] = MIN_OBSERVED_NOISE_LEVEL else: raise ValueError( "Mix of known and unknown variances indicates valuation function " "errors. Variances should all be specified, or none should be." ) if use_input_warping: warp_tf = get_warping_transform( d=X.shape[-1], task_feature=task_feature, batch_shape=X.shape[:-2], # pyre-ignore [6] ) else: warp_tf = None if fidelity_features is None: fidelity_features = [] if len(fidelity_features) == 0: # only pass linear_truncated arg if there are fidelities kwargs = {k: v for k, v in kwargs.items() if k != "linear_truncated"} if len(fidelity_features) > 0: if task_feature: raise NotImplementedError( # pragma: no cover "multi-task multi-fidelity models not yet available" ) # at this point we can assume that there is only a single fidelity parameter gp = SingleTaskMultiFidelityGP( train_X=X, train_Y=Y, data_fidelity=fidelity_features[0], input_transform=warp_tf, **kwargs, ) elif task_feature is None and all_nan_Yvar: gp = SingleTaskGP(train_X=X, train_Y=Y, input_transform=warp_tf, **kwargs) elif task_feature is None: gp = FixedNoiseGP( train_X=X, train_Y=Y, train_Yvar=Yvar, input_transform=warp_tf, **kwargs ) else: # instantiate multitask GP all_tasks, _, _ = MultiTaskGP.get_all_tasks(X, task_feature) num_tasks = len(all_tasks) prior_dict = kwargs.get("prior") prior = None if prior_dict is not None: prior_type = prior_dict.get("type", None) if issubclass(prior_type, LKJCovariancePrior): sd_prior = prior_dict.get("sd_prior", GammaPrior(1.0, 0.15)) sd_prior._event_shape = torch.Size([num_tasks]) eta = prior_dict.get("eta", 0.5) if not isinstance(eta, float) and not isinstance(eta, int): raise ValueError(f"eta must be a real number, your eta was {eta}") prior = LKJCovariancePrior(num_tasks, eta, sd_prior) else: raise NotImplementedError( "Currently only LKJ prior is supported," f"your prior type was {prior_type}." ) if all_nan_Yvar: gp = MultiTaskGP( train_X=X, train_Y=Y, task_feature=task_feature, rank=kwargs.get("rank"), task_covar_prior=prior, input_transform=warp_tf, ) else: gp = FixedNoiseMultiTaskGP( train_X=X, train_Y=Y, train_Yvar=Yvar, task_feature=task_feature, rank=kwargs.get("rank"), task_covar_prior=prior, input_transform=warp_tf, ) return gp
def test_FixedNoiseMultiTaskGP(self, cuda=False): for double in (False, True): tkwargs = { "device": torch.device("cuda") if cuda else torch.device("cpu"), "dtype": torch.double if double else torch.float, } model = _get_fixed_noise_model(**tkwargs) self.assertIsInstance(model, FixedNoiseMultiTaskGP) self.assertIsInstance(model.likelihood, FixedNoiseGaussianLikelihood) self.assertIsInstance(model.mean_module, ConstantMean) self.assertIsInstance(model.covar_module, ScaleKernel) matern_kernel = model.covar_module.base_kernel self.assertIsInstance(matern_kernel, MaternKernel) self.assertIsInstance(matern_kernel.lengthscale_prior, GammaPrior) self.assertIsInstance(model.task_covar_module, IndexKernel) self.assertEqual(model._rank, 2) self.assertEqual(model.task_covar_module.covar_factor.shape[-1], model._rank) # test model fitting mll = ExactMarginalLogLikelihood(model.likelihood, model) mll = fit_gpytorch_model(mll, options={"maxiter": 1}) # test posterior test_x = torch.rand(2, 1, **tkwargs) posterior_f = model.posterior(test_x) self.assertIsInstance(posterior_f, GPyTorchPosterior) self.assertIsInstance(posterior_f.mvn, MultitaskMultivariateNormal) self.assertEqual(posterior_f.mean.shape, torch.Size([2, 2])) self.assertEqual(posterior_f.variance.shape, torch.Size([2, 2])) # TODO: test posterior w/ observation noise # test posterior w/ single output index posterior_f = model.posterior(test_x, output_indices=[0]) self.assertIsInstance(posterior_f, GPyTorchPosterior) self.assertIsInstance(posterior_f.mvn, MultivariateNormal) self.assertEqual(posterior_f.mean.shape, torch.Size([2, 1])) self.assertEqual(posterior_f.variance.shape, torch.Size([2, 1])) # test posterior w/ bad output index with self.assertRaises(ValueError): model.posterior(test_x, output_indices=[2]) # test posterior (batch eval) test_x = torch.rand(3, 2, 1, **tkwargs) posterior_f = model.posterior(test_x) self.assertIsInstance(posterior_f, GPyTorchPosterior) self.assertIsInstance(posterior_f.mvn, MultitaskMultivariateNormal) # test that unsupported batch shape MTGPs throw correct error with self.assertRaises(ValueError): FixedNoiseMultiTaskGP(torch.rand(2, 2, 2), torch.rand(2, 1), torch.rand(2, 1), 0) # test that bad feature index throws correct error train_X, train_Y = _get_random_mt_data(**tkwargs) train_Yvar = torch.full_like(train_Y, 0.05) with self.assertRaises(ValueError): FixedNoiseMultiTaskGP(train_X, train_Y, train_Yvar, 2) # test that bad output task throws correct error with self.assertRaises(RuntimeError): FixedNoiseMultiTaskGP(train_X, train_Y, train_Yvar, 0, output_tasks=[2])