def run(params, dataloaders):
# run the inference algorithm, this returns the mean predcitions for the test data
# and the associated uncertainty estimates
device = utils.get_device()
mu,eta = inference(params, dataloaders)
return rescale_data(dataloaders, mu, eta)
def __init__(self, mu=0, sigma=0.05, device=None):
super(VarianceConstrainedAEBottleneck, self).__init__()
self.mu = mu
self.sigma = sigma
if not device:
device = get_device()
self.device = device
def __call__(self, x, x_recon, mu, log_var, distribution='gaussian'):
C = torch.clamp(self.C_max / self.C_stop_iter * self.ITERATIONS, 0,
self.C_max.data[0])
C = C.to(get_device())
recon_loss = reconstruction_loss(x, x_recon, distribution)
total_kld, dim_wise_kld, mean_kld = kl_divergence(mu, log_var)
loss = recon_loss + self.GAMMA * (total_kld - C).abs()
return loss
def __init__(self, z_dim, device=None):
super().__init__()
if not device:
device = get_device()
flows = [MAF(dim=z_dim, parity=True, nh=64) for i in range(5)]
self.prior = torch.distributions.Normal(
torch.zeros(z_dim).to(device),
torch.ones(z_dim).to(device))
self.flow = NormalizingFlow(flows).to(device)
def __init__(self, args):
self.args = args
set_seeds(self.args.seed)
if args.tokenizer == 'bert_unigram':
tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab,
do_lower_case=True)
tokenize = lambda x: tokenizer.tokenize(
tokenizer.convert_to_unicode(x))
elif args.tokenizer == 'sentencepiece':
tokenizer = tokenization.SPTokenizer(model_path=args.model_path,
nbest_size=args.sp_nbest_size,
alpha=args.sp_alpha,
vocab_file=args.vocab)
tokenize = lambda x: tokenizer.tokenize(
tokenizer.convert_to_unicode(x))
pipeline = [
Preprocess4Pretrain(args.max_pred, args.mask_prob,
list(tokenizer.vocab.keys()),
tokenizer.convert_tokens_to_ids,
self.args.max_len, args.mask_alpha,
args.mask_beta, args.max_gram)
]
data_iter = DataLoader(SentPairDataset(args.data_file,
self.args.batch_size,
tokenize,
self.args.max_len,
pipeline=pipeline),
batch_size=self.args.batch_size,
collate_fn=seq_collate,
num_workers=mp.cpu_count())
discriminator = Discriminator(self.args)
generator = Generator(self.args)
self.optimizer = optim.optim4GPU(self.args, generator, discriminator)
# self.g_optimizer = optim.optim4GPU(self.args, generator)
self.trainer = train.AdversarialTrainer(self.args, discriminator,
generator, data_iter,
self.optimizer, args.ratio,
args.save_dir, get_device())
os.makedirs(os.path.join(args.log_dir, args.name), exist_ok=True)
self.writer = SummaryWriter(log_dir=os.path.join(
args.log_dir, args.name)) # for tensorboardX
def inference(params, dataloaders, use_tqdm=True):
device = utils.get_device()
# mc dropout
predict = load_trained_model(params, device)
ymc_hats, eta_1s = mc_dropout(params, predict, dataloaders['test'], device)
# inherent noise
predict.apply(dropout_off)
for x,y in dataloaders['valid']:
x,y = x.to(device), y.to(device)
break
eta_2sq = np.mean(cpu(predict((x, y[:,0,1:])))[:,0])
# total noise
etas = np.sqrt(eta_1s + eta_2sq)
return ymc_hats, etas
def __init__(self, flows, device=None):
super().__init__()
self.flows = nn.ModuleList(flows)
if not device:
device = get_device()
self.device = device
#!/usr/bin/env python
# coding: utf-8
from torch import optim
from src.args import args
from src.features_extraction import WassersteinAE, WassersteinAELossFunction
from src.utils import get_fixed_hyper_param, train_ae, get_device, game_data_loaders
device = get_device()
batch_size, num_of_channels, input_size, z_dim = get_fixed_hyper_param(
args['hyper_parameters'])
reg_weight = args['wasserstein_ae']['reg_weight']
DO_TRAIN = True
model = WassersteinAE(z_dim, num_of_channels, input_size).to(device)
loss = WassersteinAELossFunction(reg_weight)
optimizer = optim.Adam(model.parameters())
dataloaders = game_data_loaders()
train_loaders, val_loaders = dataloaders['train'], dataloaders['val']
if DO_TRAIN:
num_epochs = int(3e3)
train_ae(num_epochs, model, dataloaders['train'], dataloaders['val'],
optimizer, device, loss)
def run_epoch(model, discriminator, iterator, encoder_optimizer, decoder_optimizer, discriminator_optimizer,
criterion, master_bar, phase='train', epoch=0, writer=None, device=None):
device = get_device() if not device else device
is_train = (phase == 'train')
if is_train:
model.train()
else:
model.eval()
epoch_reconstracted_loss = 0
epoch_discriminator_loss = 0
epoch_generator_loss = 0
with torch.set_grad_enabled(is_train):
for i, X in enumerate(progress_bar(iterator, parent=master_bar)):
global_i = len(iterator) * epoch + i
# data to device
X = X.to(device)
# make prediction
X_reconstracted, z = model(X)
# calculate loss
recon_loss = criterion(X, X_reconstracted)
if is_train:
# make optimization step
decoder_optimizer.zero_grad()
encoder_optimizer.zero_grad()
recon_loss.backward()
decoder_optimizer.step()
encoder_optimizer.step()
# Evaluation mode so dropout is off
model.encoder.eval()
# Generating samples from a Gaussian distribution
z_real = (torch.randn_like(z) * 5).to(device) # Sample from N(0,5)
# Latent code (compression of the image)
z_fake = model.encoder(X)
# Output of the Discriminator for gaussian and compressed z_dim dimensional vector
discriminator_real, discriminator_fake = discriminator(z_real), discriminator(z_fake)
# Loss of the discriminator from the template distribution
discriminator_loss = -torch.mean(
torch.log(discriminator_real + 1e-8) + torch.log(1 - discriminator_fake + 1e-8))
if is_train:
# Optimisation of the Discriminator
discriminator_optimizer.zero_grad()
discriminator_loss.backward()
discriminator_optimizer.step()
if is_train:
# Updating Generator/Encoder
model.encoder.train()
z_fake = model.encoder(X)
discriminator_fake = discriminator(z_fake)
generator_loss = -torch.mean(torch.log(discriminator_fake + 1e-8))
if is_train:
encoder_optimizer.zero_grad()
generator_loss.backward()
encoder_optimizer.step()
# dump train metrics to tensor-board
if writer is not None and is_train:
writer.add_scalar(f"loss/{phase}", recon_loss.item(), global_i)
writer.add_scalar(f"discriminator_loss/{phase}", discriminator_loss.item(), global_i)
writer.add_scalar(f"generator_loss/{phase}", generator_loss.item(), global_i)
epoch_reconstracted_loss += recon_loss.item()
epoch_discriminator_loss += discriminator_loss.item()
epoch_generator_loss += generator_loss.item()
# dump epoch metrics to tensor-board
if writer is not None:
writer.add_scalar(f"loss_epoch/{phase}", epoch_reconstracted_loss / len(iterator), epoch)
writer.add_scalar(f"discriminator_loss_epoch/{phase}", epoch_discriminator_loss / len(iterator), epoch)
writer.add_scalar(f"generator_loss_epoch/{phase}", epoch_generator_loss / len(iterator), epoch)
return epoch_reconstracted_loss / len(iterator), epoch_discriminator_loss / len(
iterator), epoch_generator_loss / len(iterator)
