def fit(self, features, labels): # tran standard SWAG model results = super().fit(features, labels) mean, var, cov = self.swag_model.get_space() if self.use_cuda: mean, cov = mean.cuda(), cov.cuda() if self.temperature is None: self.temperature = features.shape[0] / cov.size(0) print('Temperature:', self.temperature) # print(cov.size()) subspace = SubspaceModel(mean, cov) self.ess_model = EllipticalSliceSampling(self.base, subspace=subspace, var=var, loader=self.data_loader, criterion=self.criterion, num_samples=self.num_samples, use_cuda=self.use_cuda, *self.args, **self.kwargs) train_loss = self.ess_model.fit(scale=self.prior_std, use_cuda=self.use_cuda, temperature=self.temperature, minibatch=self.mb) print(train_loss)
def fit(self, features, labels): # tran standard SWAG model results = super().fit(features, labels) mean, _, cov = self.swag_model.get_space() if self.use_cuda: mean, cov = mean.cuda(), cov.cuda() subspace = SubspaceModel(mean, cov) if self.use_cuda: torch.set_default_tensor_type(torch.cuda.FloatTensor) # now form BenchmarkPyroClass self.bench_pyro = BenchmarkPyroModel( self.base, subspace, likelihood_given_outputs=self.likelihood, prior_log_sigma=self.prior_log_sigma, batch_size=self.batch_size, num_samples=self.num_samples, *self.args, **self.kwargs) if self.use_cuda: self.bench_pyro.cuda() self.features, self.labels = self.features.cuda( ), self.labels.cuda() self.bench_pyro.fit(self.features, self.labels)
def fit(self, features, labels): # tran standard SWAG model results = super().fit(features, labels) mean, _, cov = self.swag_model.get_space() if self.use_cuda: mean, cov = mean.cuda(), cov.cuda() subspace = SubspaceModel(mean, cov) if self.use_cuda: torch.set_default_tensor_type(torch.cuda.FloatTensor) if self.temperature is None: self.temperature = features.shape[0] / cov.size(0) print('Temperature:', self.temperature) self.bench_vi = BenchmarkVIModel(loader=self.data_loader, criterion=self.criterion, epochs=self.num_samples, base=self.base, subspace=subspace, prior_log_sigma=self.prior_log_sigma, temperature=self.temperature, use_cuda=self.use_cuda, *self.args, **self.kwargs) self.bench_vi.fit()
tsvd = sklearn.decomposition.TruncatedSVD(n_components=args.rank, n_iter=7, random_state=args.seed) tsvd.fit(cov_factor) cov_factor = tsvd.components_ cov_factor /= np.linalg.norm(cov_factor, axis=1, keepdims=True) cov_factor *= scale print(cov_factor[:, 0]) cov_factor = torch.FloatTensor(cov_factor, device=mean.device) vi_model = VIModel( subspace=SubspaceModel(mean.cuda(), cov_factor.cuda()), init_inv_softplus_sigma=math.log(math.exp(args.init_std) - 1.0), prior_log_sigma=math.log(args.prior_std), num_classes=num_classes, base=model_cfg.base, with_mu=not args.no_mu, *model_cfg.args, **model_cfg.kwargs) vi_model = vi_model.cuda() print(utils.eval(loaders["train"], vi_model, criterion=losses.cross_entropy)) elbo = ELBO(losses.cross_entropy, len(loaders['train'].dataset), args.temperature) #optimizer = torch.optim.Adam([param for param in vi_model.parameters()], lr=0.01)
w[1] = 0.25 * (w[0] + w[2]) + 0.5 * w[1] mean = np.mean(w, axis=0) u = w[2] - w[0] du = np.linalg.norm(u) v = w[1] - w[0] v -= u / du * np.sum(u / du * v) dv = np.linalg.norm(v) print(du, dv) u /= math.sqrt(3.0) v /= 1.5 cov_factor = np.vstack((u[None, :], v[None, :])) subspace = SubspaceModel(torch.FloatTensor(mean), torch.FloatTensor(cov_factor)) coords = np.dot( cov_factor / np.sum(np.square(cov_factor), axis=1, keepdims=True), (w - mean[None, :]).T).T print(coords) theta = torch.FloatTensor(coords[2, :]) for i in range(3): v = subspace(torch.FloatTensor(coords[i])) offset = 0 for param in model.parameters(): param.data.copy_(v[offset:offset + param.numel()].view( param.size()).to(args.device)) offset += param.numel()