def run(params, dataloaders): # run the inference algorithm, this returns the mean predcitions for the test data # and the associated uncertainty estimates device = utils.get_device() mu,eta = inference(params, dataloaders) return rescale_data(dataloaders, mu, eta)
def __init__(self, mu=0, sigma=0.05, device=None): super(VarianceConstrainedAEBottleneck, self).__init__() self.mu = mu self.sigma = sigma if not device: device = get_device() self.device = device
def __call__(self, x, x_recon, mu, log_var, distribution='gaussian'): C = torch.clamp(self.C_max / self.C_stop_iter * self.ITERATIONS, 0, self.C_max.data[0]) C = C.to(get_device()) recon_loss = reconstruction_loss(x, x_recon, distribution) total_kld, dim_wise_kld, mean_kld = kl_divergence(mu, log_var) loss = recon_loss + self.GAMMA * (total_kld - C).abs() return loss
def __init__(self, z_dim, device=None): super().__init__() if not device: device = get_device() flows = [MAF(dim=z_dim, parity=True, nh=64) for i in range(5)] self.prior = torch.distributions.Normal( torch.zeros(z_dim).to(device), torch.ones(z_dim).to(device)) self.flow = NormalizingFlow(flows).to(device)
def __init__(self, args): self.args = args set_seeds(self.args.seed) if args.tokenizer == 'bert_unigram': tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab, do_lower_case=True) tokenize = lambda x: tokenizer.tokenize( tokenizer.convert_to_unicode(x)) elif args.tokenizer == 'sentencepiece': tokenizer = tokenization.SPTokenizer(model_path=args.model_path, nbest_size=args.sp_nbest_size, alpha=args.sp_alpha, vocab_file=args.vocab) tokenize = lambda x: tokenizer.tokenize( tokenizer.convert_to_unicode(x)) pipeline = [ Preprocess4Pretrain(args.max_pred, args.mask_prob, list(tokenizer.vocab.keys()), tokenizer.convert_tokens_to_ids, self.args.max_len, args.mask_alpha, args.mask_beta, args.max_gram) ] data_iter = DataLoader(SentPairDataset(args.data_file, self.args.batch_size, tokenize, self.args.max_len, pipeline=pipeline), batch_size=self.args.batch_size, collate_fn=seq_collate, num_workers=mp.cpu_count()) discriminator = Discriminator(self.args) generator = Generator(self.args) self.optimizer = optim.optim4GPU(self.args, generator, discriminator) # self.g_optimizer = optim.optim4GPU(self.args, generator) self.trainer = train.AdversarialTrainer(self.args, discriminator, generator, data_iter, self.optimizer, args.ratio, args.save_dir, get_device()) os.makedirs(os.path.join(args.log_dir, args.name), exist_ok=True) self.writer = SummaryWriter(log_dir=os.path.join( args.log_dir, args.name)) # for tensorboardX
def inference(params, dataloaders, use_tqdm=True): device = utils.get_device() # mc dropout predict = load_trained_model(params, device) ymc_hats, eta_1s = mc_dropout(params, predict, dataloaders['test'], device) # inherent noise predict.apply(dropout_off) for x,y in dataloaders['valid']: x,y = x.to(device), y.to(device) break eta_2sq = np.mean(cpu(predict((x, y[:,0,1:])))[:,0]) # total noise etas = np.sqrt(eta_1s + eta_2sq) return ymc_hats, etas
def __init__(self, flows, device=None): super().__init__() self.flows = nn.ModuleList(flows) if not device: device = get_device() self.device = device
#!/usr/bin/env python # coding: utf-8 from torch import optim from src.args import args from src.features_extraction import WassersteinAE, WassersteinAELossFunction from src.utils import get_fixed_hyper_param, train_ae, get_device, game_data_loaders device = get_device() batch_size, num_of_channels, input_size, z_dim = get_fixed_hyper_param( args['hyper_parameters']) reg_weight = args['wasserstein_ae']['reg_weight'] DO_TRAIN = True model = WassersteinAE(z_dim, num_of_channels, input_size).to(device) loss = WassersteinAELossFunction(reg_weight) optimizer = optim.Adam(model.parameters()) dataloaders = game_data_loaders() train_loaders, val_loaders = dataloaders['train'], dataloaders['val'] if DO_TRAIN: num_epochs = int(3e3) train_ae(num_epochs, model, dataloaders['train'], dataloaders['val'], optimizer, device, loss)
def run_epoch(model, discriminator, iterator, encoder_optimizer, decoder_optimizer, discriminator_optimizer, criterion, master_bar, phase='train', epoch=0, writer=None, device=None): device = get_device() if not device else device is_train = (phase == 'train') if is_train: model.train() else: model.eval() epoch_reconstracted_loss = 0 epoch_discriminator_loss = 0 epoch_generator_loss = 0 with torch.set_grad_enabled(is_train): for i, X in enumerate(progress_bar(iterator, parent=master_bar)): global_i = len(iterator) * epoch + i # data to device X = X.to(device) # make prediction X_reconstracted, z = model(X) # calculate loss recon_loss = criterion(X, X_reconstracted) if is_train: # make optimization step decoder_optimizer.zero_grad() encoder_optimizer.zero_grad() recon_loss.backward() decoder_optimizer.step() encoder_optimizer.step() # Evaluation mode so dropout is off model.encoder.eval() # Generating samples from a Gaussian distribution z_real = (torch.randn_like(z) * 5).to(device) # Sample from N(0,5) # Latent code (compression of the image) z_fake = model.encoder(X) # Output of the Discriminator for gaussian and compressed z_dim dimensional vector discriminator_real, discriminator_fake = discriminator(z_real), discriminator(z_fake) # Loss of the discriminator from the template distribution discriminator_loss = -torch.mean( torch.log(discriminator_real + 1e-8) + torch.log(1 - discriminator_fake + 1e-8)) if is_train: # Optimisation of the Discriminator discriminator_optimizer.zero_grad() discriminator_loss.backward() discriminator_optimizer.step() if is_train: # Updating Generator/Encoder model.encoder.train() z_fake = model.encoder(X) discriminator_fake = discriminator(z_fake) generator_loss = -torch.mean(torch.log(discriminator_fake + 1e-8)) if is_train: encoder_optimizer.zero_grad() generator_loss.backward() encoder_optimizer.step() # dump train metrics to tensor-board if writer is not None and is_train: writer.add_scalar(f"loss/{phase}", recon_loss.item(), global_i) writer.add_scalar(f"discriminator_loss/{phase}", discriminator_loss.item(), global_i) writer.add_scalar(f"generator_loss/{phase}", generator_loss.item(), global_i) epoch_reconstracted_loss += recon_loss.item() epoch_discriminator_loss += discriminator_loss.item() epoch_generator_loss += generator_loss.item() # dump epoch metrics to tensor-board if writer is not None: writer.add_scalar(f"loss_epoch/{phase}", epoch_reconstracted_loss / len(iterator), epoch) writer.add_scalar(f"discriminator_loss_epoch/{phase}", epoch_discriminator_loss / len(iterator), epoch) writer.add_scalar(f"generator_loss_epoch/{phase}", epoch_generator_loss / len(iterator), epoch) return epoch_reconstracted_loss / len(iterator), epoch_discriminator_loss / len( iterator), epoch_generator_loss / len(iterator)