def _get_model(n, fixed_noise=False, use_octf=False, **tkwargs): train_x1, train_y1 = _get_random_data(batch_shape=torch.Size(), m=1, n=10, **tkwargs) train_x2, train_y2 = _get_random_data(batch_shape=torch.Size(), m=1, n=11, **tkwargs) octfs = [Standardize(m=1), Standardize(m=1)] if use_octf else [None, None] if fixed_noise: train_y1_var = 0.1 + 0.1 * torch.rand_like(train_y1, **tkwargs) train_y2_var = 0.1 + 0.1 * torch.rand_like(train_y2, **tkwargs) model1 = FixedNoiseGP( train_X=train_x1, train_Y=train_y1, train_Yvar=train_y1_var, outcome_transform=octfs[0], ) model2 = FixedNoiseGP( train_X=train_x2, train_Y=train_y2, train_Yvar=train_y2_var, outcome_transform=octfs[1], ) else: model1 = SingleTaskGP(train_X=train_x1, train_Y=train_y1, outcome_transform=octfs[0]) model2 = SingleTaskGP(train_X=train_x2, train_Y=train_y2, outcome_transform=octfs[1]) model = ModelListGP(model1, model2) return model.to(**tkwargs)
def _get_model_and_data(self, batch_shape, num_outputs, **tkwargs): train_X, train_Y = _get_random_data(batch_shape=batch_shape, num_outputs=num_outputs, **tkwargs) model_kwargs = {"train_X": train_X, "train_Y": train_Y} model = SingleTaskGP(**model_kwargs) return model, model_kwargs
def _get_model(n, fixed_noise=False, **tkwargs): train_x1, train_x2, train_y1, train_y2 = _get_random_data(n=n, **tkwargs) if fixed_noise: train_y1_var = 0.1 + 0.1 * torch.rand_like(train_y1, **tkwargs) train_y2_var = 0.1 + 0.1 * torch.rand_like(train_y2, **tkwargs) model1 = FixedNoiseGP(train_X=train_x1, train_Y=train_y1, train_Yvar=train_y1_var) model2 = FixedNoiseGP(train_X=train_x2, train_Y=train_y2, train_Yvar=train_y2_var) else: model1 = SingleTaskGP(train_X=train_x1, train_Y=train_y1) model2 = SingleTaskGP(train_X=train_x2, train_Y=train_y2) model = ModelListGP(model1, model2) return model.to(**tkwargs)
def initialize_model(train_x, train_obj): # define models for objective and constraint model = SingleTaskGP(train_x, train_obj, outcome_transform=Standardize(m=train_obj.shape[-1])) mll = ExactMarginalLogLikelihood(model.likelihood, model) return mll, model
def _get_model( X: Tensor, Y: Tensor, Yvar: Tensor, task_feature: Optional[int] = None, fidelity_features: Optional[List[int]] = None, **kwargs: Any, ) -> GPyTorchModel: """Instantiate a model of type depending on the input data. Args: X: A `n x d` tensor of input features. Y: A `n x m` tensor of input observations. Yvar: A `n x m` tensor of input variances (NaN if unobserved). task_feature: The index of the column pertaining to the task feature (if present). fidelity_features: List of columns of X that are fidelity parameters. Returns: A GPyTorchModel (unfitted). """ Yvar = Yvar.clamp_min_(MIN_OBSERVED_NOISE_LEVEL) is_nan = torch.isnan(Yvar) any_nan_Yvar = torch.any(is_nan) all_nan_Yvar = torch.all(is_nan) if any_nan_Yvar and not all_nan_Yvar: raise ValueError( "Mix of known and unknown variances indicates valuation function " "errors. Variances should all be specified, or none should be." ) if fidelity_features is None: fidelity_features = [] if len(fidelity_features) == 0: # only pass linear_truncated arg if there are fidelities kwargs = {k: v for k, v in kwargs.items() if k != "linear_truncated"} if len(fidelity_features) > 0: if task_feature: raise NotImplementedError( "multi-task multi-fidelity models not yet available" ) # at this point we can assume that there is only a single fidelity parameter gp = SingleTaskMultiFidelityGP( train_X=X, train_Y=Y, data_fidelity=fidelity_features[0], **kwargs ) elif task_feature is None and all_nan_Yvar: gp = SingleTaskGP(train_X=X, train_Y=Y, **kwargs) elif task_feature is None: gp = FixedNoiseGP(train_X=X, train_Y=Y, train_Yvar=Yvar, **kwargs) elif all_nan_Yvar: gp = MultiTaskGP(train_X=X, train_Y=Y, task_feature=task_feature, **kwargs) else: gp = FixedNoiseMultiTaskGP( train_X=X, train_Y=Y.view(-1), train_Yvar=Yvar.view(-1), task_feature=task_feature, **kwargs, ) return gp
def _get_model(self, batch_shape, num_outputs, likelihood=None, **tkwargs): train_x, train_y = _get_random_data(batch_shape=batch_shape, num_outputs=num_outputs, **tkwargs) model = SingleTaskGP(train_X=train_x, train_Y=train_y, likelihood=likelihood) mll = ExactMarginalLogLikelihood(model.likelihood, model).to(**tkwargs) fit_gpytorch_model(mll, options={"maxiter": 1}) return model
def __init__(self, stem, init_x, init_y, lr, **kwargs): super().__init__() self.stem = stem.to(init_x.device) if init_y.t().shape[0] != 1: _batch_shape = init_y.t().shape[:-1] else: _batch_shape = torch.Size() features = self.stem(init_x) self.gp = SingleTaskGP(features, init_y, covar_module=ScaleKernel( RBFKernel(batch_shape=_batch_shape, ard_num_dims=stem.output_dim), batch_shape=_batch_shape)) self.mll = ExactMarginalLogLikelihood(self.gp.likelihood, self.gp) self.optimizer = torch.optim.Adam(self.parameters(), lr=lr) self._raw_inputs = [init_x] self._target_batch_shape = _batch_shape self.target_dim = init_y.size(-1)
def test_ModelListGP_single(self): tkwargs = {"device": self.device, "dtype": torch.float} train_x1, train_x2, train_y1, train_y2 = _get_random_data(n=10, **tkwargs) model1 = SingleTaskGP(train_X=train_x1, train_Y=train_y1) model = ModelListGP(model1) model.to(**tkwargs) test_x = torch.tensor([[0.25], [0.75]], **tkwargs) posterior = model.posterior(test_x) self.assertIsInstance(posterior, GPyTorchPosterior) self.assertIsInstance(posterior.mvn, MultivariateNormal)
def test_set_transformed_inputs(self): # This intended to catch https://github.com/pytorch/botorch/issues/1078. # More general testing of _set_transformed_inputs is done under ModelListGP. X = torch.rand(5, 2) Y = X**2 for tf_class in [Normalize, InputStandardize]: intf = tf_class(d=2) model = SingleTaskGP(X, Y, input_transform=intf) mll = ExactMarginalLogLikelihood(model.likelihood, model) fit_gpytorch_model(mll, options={"maxiter": 2}) tf_X = intf(X) self.assertEqual(X.shape, tf_X.shape)
def _get_model_and_data( self, batch_shape, m, outcome_transform=None, input_transform=None, **tkwargs ): train_X, train_Y = _get_random_data(batch_shape=batch_shape, m=m, **tkwargs) model_kwargs = { "train_X": train_X, "train_Y": train_Y, "outcome_transform": outcome_transform, "input_transform": input_transform, } model = SingleTaskGP(**model_kwargs) return model, model_kwargs
def _get_model_and_data(self, batch_shape, m, outcome_transform=None, **tkwargs): train_X, train_Y = _get_random_data(batch_shape=batch_shape, num_outputs=m, **tkwargs) model_kwargs = {"train_X": train_X, "train_Y": train_Y} if outcome_transform is not None: model_kwargs["outcome_transform"] = outcome_transform model = SingleTaskGP(**model_kwargs) return model, model_kwargs
def test_ModelListGPSingle(self, cuda=False): tkwargs = { "device": torch.device("cuda") if cuda else torch.device("cpu"), "dtype": torch.float, } train_x1, train_x2, train_y1, train_y2 = _get_random_data(n=10, **tkwargs) model1 = SingleTaskGP(train_X=train_x1, train_Y=train_y1) model = ModelListGP(gp_models=[model1]) model.to(**tkwargs) test_x = (torch.tensor([0.25, 0.75]).type_as(model.train_targets[0]),) posterior = model.posterior(test_x) self.assertIsInstance(posterior, GPyTorchPosterior) self.assertIsInstance(posterior.mvn, MultivariateNormal)
def _get_model( X: Tensor, Y: Tensor, Yvar: Tensor, task_feature: Optional[int], fidelity_features: Optional[List[int]] = None, fidelity_model_id: Optional[int] = None, ) -> GPyTorchModel: """Instantiate a model of type depending on the input data.""" Yvar = Yvar.clamp_min_(MIN_OBSERVED_NOISE_LEVEL) # pyre-ignore: [16] is_nan = torch.isnan(Yvar) any_nan_Yvar = torch.any(is_nan) all_nan_Yvar = torch.all(is_nan) if any_nan_Yvar and not all_nan_Yvar: raise ValueError( "Mix of known and unknown variances indicates " "valuation function errors. Variances should all be specified, or " "none should be.") if fidelity_features is None: fidelity_features = [] if fidelity_model_id is None or len(fidelity_features) == 0: if task_feature is None and all_nan_Yvar: gp = SingleTaskGP(train_X=X, train_Y=Y) elif task_feature is None: gp = FixedNoiseGP(train_X=X, train_Y=Y, train_Yvar=Yvar) elif all_nan_Yvar: gp = MultiTaskGP(train_X=X, train_Y=Y, task_feature=task_feature) else: gp = FixedNoiseMultiTaskGP( train_X=X, train_Y=Y.view(-1), train_Yvar=Yvar.view(-1), task_feature=task_feature, ) else: gp_model = model_list[fidelity_model_id] # pyre-ignore [29] gp = gp_model(train_X=X, train_Y=Y, train_data_fidelity=False) return gp
def singletask_gp_model_constructor( Xs: List[Tensor], Ys: List[Tensor], Yvars: List[Tensor], task_features: List[int], fidelity_features: List[int], metric_names: List[str], state_dict: Optional[Dict[str, Tensor]] = None, refit_model: bool = True, **kwargs: Any, ) -> Model: gp = SingleTaskGP(train_X=Xs[0], train_Y=Ys[0], **kwargs) gp.to(Xs[0]) if state_dict is not None: gp.load_state_dict(state_dict) if state_dict is None or refit_model: fit_gpytorch_model(ExactMarginalLogLikelihood(gp.likelihood, gp)) return gp
def test_sample_cached_cholesky(self): torch.manual_seed(0) tkwargs = {"device": self.device} for dtype in (torch.float, torch.double): tkwargs["dtype"] = dtype train_X = torch.rand(10, 2, **tkwargs) train_Y = torch.randn(10, 2, **tkwargs) for m in (1, 2): model_list_values = (True, False) if m == 2 else (False, ) for use_model_list in model_list_values: if use_model_list: model = ModelListGP( SingleTaskGP( train_X, train_Y[..., :1], ), SingleTaskGP( train_X, train_Y[..., 1:], ), ) else: model = SingleTaskGP( train_X, train_Y[:, :m], ) sampler = IIDNormalSampler(3) base_sampler = IIDNormalSampler(3) for q in (1, 3, 9): # test batched baseline_L for train_batch_shape in ( torch.Size([]), torch.Size([3]), torch.Size([3, 2]), ): # test batched test points for test_batch_shape in ( torch.Size([]), torch.Size([4]), torch.Size([4, 2]), ): if len(train_batch_shape) > 0: train_X_ex = train_X.unsqueeze(0).expand( train_batch_shape + train_X.shape) else: train_X_ex = train_X if len(test_batch_shape) > 0: test_X = train_X_ex.unsqueeze(0).expand( test_batch_shape + train_X_ex.shape) else: test_X = train_X_ex with torch.no_grad(): base_posterior = model.posterior( train_X_ex[..., :-q, :]) mvn = base_posterior.mvn lazy_covar = mvn.lazy_covariance_matrix if m == 2: lazy_covar = lazy_covar.base_lazy_tensor baseline_L = lazy_covar.root_decomposition( ) baseline_L = baseline_L.root.evaluate() test_X = test_X.clone().requires_grad_(True) new_posterior = model.posterior(test_X) samples = sampler(new_posterior) samples[..., -q:, :].sum().backward() test_X2 = test_X.detach().clone( ).requires_grad_(True) new_posterior2 = model.posterior(test_X2) q_samples = sample_cached_cholesky( posterior=new_posterior2, baseline_L=baseline_L, q=q, base_samples=sampler.base_samples.detach(). clone(), sample_shape=sampler.sample_shape, ) q_samples.sum().backward() all_close_kwargs = ({ "atol": 1e-4, "rtol": 1e-2, } if dtype == torch.float else {}) self.assertTrue( torch.allclose( q_samples.detach(), samples[..., -q:, :].detach(), **all_close_kwargs, )) self.assertTrue( torch.allclose( test_X2.grad[..., -q:, :], test_X.grad[..., -q:, :], **all_close_kwargs, )) # Test that adding a new point and base_sample # did not change posterior samples for previous points. # This tests that we properly account for not # interleaving. base_sampler.base_samples = ( sampler.base_samples[ ..., :-q, :].detach().clone()) baseline_samples = base_sampler(base_posterior) new_batch_shape = samples.shape[ 1:-baseline_samples.ndim + 1] expanded_baseline_samples = baseline_samples.view( baseline_samples.shape[0], *[1] * len(new_batch_shape), *baseline_samples.shape[1:], ).expand( baseline_samples.shape[0], *new_batch_shape, *baseline_samples.shape[1:], ) self.assertTrue( torch.allclose( expanded_baseline_samples, samples[..., :-q, :], **all_close_kwargs, )) # test nans with torch.no_grad(): test_posterior = model.posterior(test_X2) test_posterior.mvn.loc = torch.full_like( test_posterior.mvn.loc, float("nan")) with self.assertRaises(NanError): sample_cached_cholesky( posterior=test_posterior, baseline_L=baseline_L, q=q, base_samples=sampler.base_samples.detach(). clone(), sample_shape=sampler.sample_shape, ) # test infs test_posterior.mvn.loc = torch.full_like( test_posterior.mvn.loc, float("inf")) with self.assertRaises(NanError): sample_cached_cholesky( posterior=test_posterior, baseline_L=baseline_L, q=q, base_samples=sampler.base_samples.detach(). clone(), sample_shape=sampler.sample_shape, ) # test triangular solve raising RuntimeError test_posterior.mvn.loc = torch.full_like( test_posterior.mvn.loc, 0.0) base_samples = sampler.base_samples.detach().clone( ) with mock.patch( "botorch.utils.low_rank.torch.triangular_solve", side_effect=RuntimeError("singular"), ): with self.assertRaises(NotPSDError): sample_cached_cholesky( posterior=test_posterior, baseline_L=baseline_L, q=q, base_samples=base_samples, sample_shape=sampler.sample_shape, ) with mock.patch( "botorch.utils.low_rank.torch.triangular_solve", side_effect=RuntimeError(""), ): with self.assertRaises(RuntimeError): sample_cached_cholesky( posterior=test_posterior, baseline_L=baseline_L, q=q, base_samples=base_samples, sample_shape=sampler.sample_shape, )
class OnlineExactRegression(torch.nn.Module): def __init__(self, stem, init_x, init_y, lr, **kwargs): super().__init__() self.stem = stem.to(init_x.device) if init_y.t().shape[0] != 1: _batch_shape = init_y.t().shape[:-1] else: _batch_shape = torch.Size() features = self.stem(init_x) self.gp = SingleTaskGP(features, init_y, covar_module=ScaleKernel( RBFKernel(batch_shape=_batch_shape, ard_num_dims=stem.output_dim), batch_shape=_batch_shape)) self.mll = ExactMarginalLogLikelihood(self.gp.likelihood, self.gp) self.optimizer = torch.optim.Adam(self.parameters(), lr=lr) self._raw_inputs = [init_x] self._target_batch_shape = _batch_shape self.target_dim = init_y.size(-1) def update(self, inputs, targets, update_stem=True, update_gp=True): inputs = inputs.view(-1, self.stem.input_dim) targets = targets.view(-1, self.target_dim) # add observation self.train() self._raw_inputs = [torch.cat([*self._raw_inputs, inputs])] self.gp.train_targets = torch.cat( [self.gp.train_targets, self._reshape_targets(targets)], dim=-1) if update_stem: self._refresh_features(*self._raw_inputs, strict=False) else: with torch.no_grad(): self._refresh_features(*self._raw_inputs, strict=False) self.mll = ExactMarginalLogLikelihood(self.gp.likelihood, self.gp) # update stem and GP if update_gp: self.optimizer.zero_grad() with gpytorch.settings.skip_logdet_forward(True): train_dist = self.gp(*self.gp.train_inputs) loss = -self.mll(train_dist, self.gp.train_targets).sum() loss.backward() self.optimizer.step() self.gp.zero_grad() # update GP training data again if update_stem: with torch.no_grad(): self._refresh_features(*self._raw_inputs) self.eval() stem_loss = gp_loss = loss.item() if update_gp else 0. return stem_loss, gp_loss def fit(self, inputs, targets, num_epochs, test_dataset=None): records = [] self.gp.train_targets = self._reshape_targets(targets) lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( self.optimizer, num_epochs, 1e-4) for epoch in range(num_epochs): self.train() self.mll.train() self.optimizer.zero_grad() self._refresh_features(inputs) train_dist = self.gp(*self.gp.train_inputs) with gpytorch.settings.skip_logdet_forward(False): loss = -self.mll(train_dist, self.gp.train_targets).sum() loss.backward() self.optimizer.step() lr_scheduler.step() self.gp.zero_grad() rmse = nll = float('NaN') if test_dataset is not None: test_x, test_y = test_dataset[:] rmse, nll = self.evaluate(test_x, test_y) records.append({ 'train_loss': loss.item(), 'test_rmse': rmse, 'test_nll': nll, 'noise': self.gp.likelihood.noise.mean().item(), 'epoch': epoch + 1 }) with torch.no_grad(): self._refresh_features(inputs) self.eval() return records def forward(self, inputs): inputs = inputs.view(-1, self.stem.input_dim) features = self.stem(inputs) return self.gp(features) def predict(self, inputs): self.eval() pred_dist = self(inputs) pred_dist = self.gp.likelihood(pred_dist) return pred_dist.mean, pred_dist.variance def evaluate(self, inputs, targets): inputs = inputs.view(-1, self.stem.input_dim) targets = targets.view(-1, self.target_dim) with torch.no_grad(): return regression.evaluate(self, inputs, targets) def set_train_data(self, inputs, targets, strict): inputs = inputs.expand(*self._target_batch_shape, -1, -1) if self.target_dim == 1: targets = targets.squeeze(0) self.gp.set_train_data(inputs, targets, strict) def _reshape_targets(self, targets): targets = targets.view(-1, self.target_dim) if targets.size(-1) == 1: targets = targets.squeeze(-1) else: targets = targets.t() return targets def _refresh_features(self, inputs, strict=True): features = self.stem(inputs) self.set_train_data(features, self.gp.train_targets, strict) return features def set_lr(self, gp_lr, stem_lr=None, bn_mom=None): stem_lr = gp_lr if stem_lr is None else stem_lr self.optimizer = torch.optim.Adam([ dict(params=self.gp.parameters(), lr=gp_lr), dict(params=self.stem.parameters(), lr=stem_lr) ]) if bn_mom is not None: for m in self.stem.modules(): if isinstance(m, torch.nn.BatchNorm1d): m.momentum = bn_mom @property def noise(self): return self.gp.likelihood.noise
def test_multi_objective_max_value_entropy(self): for dtype, m in product((torch.float, torch.double), (2, 3)): torch.manual_seed(7) # test batched model train_X = torch.rand(1, 1, 2, dtype=dtype, device=self.device) train_Y = torch.rand(1, 1, m, dtype=dtype, device=self.device) model = SingleTaskGP(train_X, train_Y) with self.assertRaises(NotImplementedError): qMultiObjectiveMaxValueEntropy(model, dummy_sample_pareto_frontiers) # test initialization train_X = torch.rand(4, 2, dtype=dtype, device=self.device) train_Y = torch.rand(4, m, dtype=dtype, device=self.device) # test batched MO model model = SingleTaskGP(train_X, train_Y) mesmo = qMultiObjectiveMaxValueEntropy( model, dummy_sample_pareto_frontiers) self.assertEqual(mesmo.num_fantasies, 16) self.assertIsInstance(mesmo.sampler, SobolQMCNormalSampler) self.assertEqual(mesmo.sampler.sample_shape, torch.Size([512])) self.assertIsInstance(mesmo.fantasies_sampler, SobolQMCNormalSampler) self.assertEqual(mesmo.posterior_max_values.shape, torch.Size([3, 1, m])) # test conversion to single-output model self.assertIs(mesmo.mo_model, model) self.assertEqual(mesmo.mo_model.num_outputs, m) self.assertIsInstance(mesmo.model, SingleTaskGP) self.assertEqual(mesmo.model.num_outputs, 1) self.assertEqual(mesmo.model._aug_batch_shape, mesmo.model._input_batch_shape) # test ModelListGP model = ModelListGP( * [SingleTaskGP(train_X, train_Y[:, i:i + 1]) for i in range(m)]) mock_sample_pfs = mock.Mock() mock_sample_pfs.return_value = dummy_sample_pareto_frontiers( model=model) mesmo = qMultiObjectiveMaxValueEntropy(model, mock_sample_pfs) self.assertEqual(mesmo.num_fantasies, 16) self.assertIsInstance(mesmo.sampler, SobolQMCNormalSampler) self.assertEqual(mesmo.sampler.sample_shape, torch.Size([512])) self.assertIsInstance(mesmo.fantasies_sampler, SobolQMCNormalSampler) self.assertEqual(mesmo.posterior_max_values.shape, torch.Size([3, 1, m])) # test conversion to batched MO model self.assertIsInstance(mesmo.mo_model, SingleTaskGP) self.assertEqual(mesmo.mo_model.num_outputs, m) self.assertIs(mesmo.mo_model, mesmo._init_model) # test conversion to single-output model self.assertIsInstance(mesmo.model, SingleTaskGP) self.assertEqual(mesmo.model.num_outputs, 1) self.assertEqual(mesmo.model._aug_batch_shape, mesmo.model._input_batch_shape) # test that we call sample_pareto_frontiers with the multi-output model mock_sample_pfs.assert_called_once_with(mesmo.mo_model) # test basic evaluation X = torch.rand(1, 2, device=self.device, dtype=dtype) with torch.no_grad(): vals = mesmo(X) igs = qMaxValueEntropy.forward(mesmo, X=X.view(1, 1, 1, 2)) self.assertEqual(vals.shape, torch.Size([1])) self.assertTrue(torch.equal(vals, igs.sum(dim=-1))) # test batched evaluation X = torch.rand(4, 1, 2, device=self.device, dtype=dtype) with torch.no_grad(): vals = mesmo(X) igs = qMaxValueEntropy.forward(mesmo, X=X.view(4, 1, 1, 2)) self.assertEqual(vals.shape, torch.Size([4])) self.assertTrue(torch.equal(vals, igs.sum(dim=-1))) # test set X pending to None mesmo.set_X_pending(None) self.assertIs(mesmo.mo_model, mesmo._init_model) fant_X = torch.cat( [ train_X.expand(16, 4, 2), torch.rand(16, 1, 2), ], dim=1, ) fant_Y = torch.cat( [ train_Y.expand(16, 4, m), torch.rand(16, 1, m), ], dim=1, ) fantasy_model = SingleTaskGP(fant_X, fant_Y) # test with X_pending is not None with mock.patch.object( SingleTaskGP, "fantasize", return_value=fantasy_model) as mock_fantasize: qMultiObjectiveMaxValueEntropy( model, dummy_sample_pareto_frontiers, X_pending=torch.rand(1, 2, device=self.device, dtype=dtype), ) mock_fantasize.assert_called_once()
def _get_model(dtype, device, multi_output=False): tkwargs = {"dtype": dtype, "device": device} train_X = torch.tensor( [ [-0.1000], [0.4894], [1.0788], [1.6681], [2.2575], [2.8469], [3.4363], [4.0257], [4.6150], [5.2044], [5.7938], [6.3832], ], **tkwargs, ) train_Y = torch.tensor( [ [-0.0274], [0.2612], [0.8114], [1.1916], [1.4870], [0.8611], [-0.9226], [-0.5916], [-1.3301], [-1.8847], [0.0647], [1.0900], ], **tkwargs, ) state_dict = { "likelihood.noise_covar.raw_noise": torch.tensor([0.0214], **tkwargs), "likelihood.noise_covar.noise_prior.concentration": torch.tensor(1.1000, **tkwargs), "likelihood.noise_covar.noise_prior.rate": torch.tensor(0.0500, **tkwargs), "mean_module.constant": torch.tensor([0.1398], **tkwargs), "covar_module.raw_outputscale": torch.tensor(0.6933, **tkwargs), "covar_module.base_kernel.raw_lengthscale": torch.tensor([[-0.0444]], **tkwargs), "covar_module.base_kernel.lengthscale_prior.concentration": torch.tensor(3.0, **tkwargs), "covar_module.base_kernel.lengthscale_prior.rate": torch.tensor(6.0, **tkwargs), "covar_module.outputscale_prior.concentration": torch.tensor(2.0, **tkwargs), "covar_module.outputscale_prior.rate": torch.tensor(0.1500, **tkwargs), } if multi_output: train_Y2 = torch.tensor( [ [0.9723], [1.0652], [0.7667], [-0.5542], [-0.6266], [-0.5350], [-0.8854], [-1.3024], [1.0408], [0.2485], [1.4924], [1.5393], ], **tkwargs, ) train_Y = torch.cat([train_Y, train_Y2], dim=-1) state_dict["likelihood.noise_covar.raw_noise"] = torch.stack([ state_dict["likelihood.noise_covar.raw_noise"], torch.tensor([0.0745], **tkwargs), ]) state_dict["mean_module.constant"] = torch.stack([ state_dict["mean_module.constant"], torch.tensor([0.3276], **tkwargs) ]) state_dict["covar_module.raw_outputscale"] = torch.stack( [ state_dict["covar_module.raw_outputscale"], torch.tensor(0.4394, **tkwargs), ], dim=-1, ) state_dict["covar_module.base_kernel.raw_lengthscale"] = torch.stack([ state_dict["covar_module.base_kernel.raw_lengthscale"], torch.tensor([[-0.4617]], **tkwargs), ]) model = SingleTaskGP(train_X, train_Y) model.load_state_dict(state_dict) return model, train_X, train_Y
def _get_model( X: Tensor, Y: Tensor, Yvar: Tensor, task_feature: Optional[int] = None, fidelity_features: Optional[List[int]] = None, use_input_warping: bool = False, **kwargs: Any, ) -> GPyTorchModel: """Instantiate a model of type depending on the input data. Args: X: A `n x d` tensor of input features. Y: A `n x m` tensor of input observations. Yvar: A `n x m` tensor of input variances (NaN if unobserved). task_feature: The index of the column pertaining to the task feature (if present). fidelity_features: List of columns of X that are fidelity parameters. Returns: A GPyTorchModel (unfitted). """ Yvar = Yvar.clamp_min(MIN_OBSERVED_NOISE_LEVEL) # pyre-ignore[16] is_nan = torch.isnan(Yvar) any_nan_Yvar = torch.any(is_nan) all_nan_Yvar = torch.all(is_nan) if any_nan_Yvar and not all_nan_Yvar: if task_feature: # TODO (jej): Replace with inferred noise before making perf judgements. Yvar[Yvar != Yvar] = MIN_OBSERVED_NOISE_LEVEL else: raise ValueError( "Mix of known and unknown variances indicates valuation function " "errors. Variances should all be specified, or none should be." ) if use_input_warping: warp_tf = get_warping_transform( d=X.shape[-1], task_feature=task_feature, batch_shape=X.shape[:-2], # pyre-ignore [6] ) else: warp_tf = None if fidelity_features is None: fidelity_features = [] if len(fidelity_features) == 0: # only pass linear_truncated arg if there are fidelities kwargs = {k: v for k, v in kwargs.items() if k != "linear_truncated"} if len(fidelity_features) > 0: if task_feature: raise NotImplementedError( # pragma: no cover "multi-task multi-fidelity models not yet available" ) # at this point we can assume that there is only a single fidelity parameter gp = SingleTaskMultiFidelityGP( train_X=X, train_Y=Y, data_fidelity=fidelity_features[0], input_transform=warp_tf, **kwargs, ) elif task_feature is None and all_nan_Yvar: gp = SingleTaskGP(train_X=X, train_Y=Y, input_transform=warp_tf, **kwargs) elif task_feature is None: gp = FixedNoiseGP( train_X=X, train_Y=Y, train_Yvar=Yvar, input_transform=warp_tf, **kwargs ) else: # instantiate multitask GP all_tasks, _, _ = MultiTaskGP.get_all_tasks(X, task_feature) num_tasks = len(all_tasks) prior_dict = kwargs.get("prior") prior = None if prior_dict is not None: prior_type = prior_dict.get("type", None) if issubclass(prior_type, LKJCovariancePrior): sd_prior = prior_dict.get("sd_prior", GammaPrior(1.0, 0.15)) sd_prior._event_shape = torch.Size([num_tasks]) eta = prior_dict.get("eta", 0.5) if not isinstance(eta, float) and not isinstance(eta, int): raise ValueError(f"eta must be a real number, your eta was {eta}") prior = LKJCovariancePrior(num_tasks, eta, sd_prior) else: raise NotImplementedError( "Currently only LKJ prior is supported," f"your prior type was {prior_type}." ) if all_nan_Yvar: gp = MultiTaskGP( train_X=X, train_Y=Y, task_feature=task_feature, rank=kwargs.get("rank"), task_covar_prior=prior, input_transform=warp_tf, ) else: gp = FixedNoiseMultiTaskGP( train_X=X, train_Y=Y, train_Yvar=Yvar, task_feature=task_feature, rank=kwargs.get("rank"), task_covar_prior=prior, input_transform=warp_tf, ) return gp
def _get_model(n, **tkwargs): train_x1, train_x2, train_y1, train_y2 = _get_random_data(n=n, **tkwargs) model1 = SingleTaskGP(train_X=train_x1, train_Y=train_y1) model2 = SingleTaskGP(train_X=train_x2, train_Y=train_y2) model = ModelListGP(gp_models=[model1, model2]) return model.to(**tkwargs)