optimizer.zero_grad()
outputs = Variable(torch.zeros(inputs.shape[0], net.num_classes, flags.train_ens).cuda())
for j in range(flags.train_ens):
outputs[:, :, j] = F.log_softmax(net(inputs), dim=1)
log_outputs = utils.logmeanexp(outputs, dim=2)
loss = criterion(log_outputs, labels)
loss.backward()
optimizer.step()
accs.append(metrics.logit2acc(log_outputs.data, labels))
training_loss += loss.cpu().data.numpy()[0]
logger.add(epoch, tr_loss=training_loss/steps, tr_acc=np.mean(accs))
# Ens 100 test
net.train()
acc, nll = utils.evaluate(net, testloader, num_ens=100)
logger.add(epoch, te_nll_ens100=nll, te_acc_ens100=acc)
# Stochastic test
net.train()
acc, nll = utils.evaluate(net, testloader, num_ens=1)
logger.add(epoch, te_nll_stoch=nll, te_acc_stoch=acc)
# Test-time averaging
net.train()
acc, nll = utils.evaluate(net, testloader, num_ens=10)
logger.add(epoch, te_nll_ens10=nll, te_acc_ens10=acc)
accs = []
steps = 0
for i, (inputs, labels) in enumerate(trainloader, 0):
steps += 1
inputs, labels = Variable(inputs.cuda(async=True)), Variable(labels.cuda(async=True))
optimizer.zero_grad()
outputs = net(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
accs.append(metrics.logit2acc(outputs.data, labels)) # probably a bad way to calculate accuracy
training_loss += loss.cpu().data.numpy()[0]
logger.add(epoch, tr_loss=training_loss/steps, tr_acc=np.mean(accs))
# Deterministic test
net.eval()
acc, nll = utils.evaluate(net, testloader, num_ens=1)
logger.add(epoch, te_nll_det=nll, te_acc_det=acc)
# Stochastic test
net.train()
acc, nll = utils.evaluate(net, testloader, num_ens=1)
logger.add(epoch, te_nll_stoch=nll, te_acc_stoch=acc)
# Test-time averaging
net.train()
acc, nll = utils.evaluate(net, testloader, num_ens=20)
logger.add(epoch, te_nll_ens=nll, te_acc_ens=acc)
optimizer.step()
elbo.append(-loss.item())
cat_mean.append(categorical_mean.item())
kls.append(kl.item() if isinstance(kl, torch.Tensor) else kl)
accuracy.append(pred2acc(pred, y_train, args.batch_size))
t1 = time() - t0
elbo = np.mean(elbo)
cat_mean = np.mean(cat_mean)
kl = np.mean(kls)
accuracy = np.mean(accuracy)
logger.add(epoch,
kl=kl,
tr_elbo=elbo,
tr_acc=accuracy,
tr_ll=cat_mean,
tr_time=t1)
model.eval()
t_dvi = time()
test_acc_dvi = evaluate(model, test_loader, mode='dvi', args=args)
t_dvi = time() - t_dvi
if not args.no_mc:
t_mc = time()
test_acc_mcvi = evaluate(model,
test_loader,
mode='mcvi',
args=args)
t_mc = time() - t_mc
def main():
fmt = {
'tr_loss': '3.1e',
'tr_acc': '.4f',
'te_acc_det': '.4f',
'te_acc_stoch': '.4f',
'te_acc_ens': '.4f',
'te_acc_perm_sigma': '.4f',
'te_acc_zero_mean': '.4f',
'te_acc_perm_sigma_ens': '.4f',
'te_acc_zero_mean_ens': '.4f',
'te_nll_det': '.4f',
'te_nll_stoch': '.4f',
'te_nll_ens': '.4f',
'te_nll_perm_sigma': '.4f',
'te_nll_zero_mean': '.4f',
'te_nll_perm_sigma_ens': '.4f',
'te_nll_zero_mean_ens': '.4f',
'time': '.3f'
}
fmt = {**fmt, **{'la%d' % i: '.4f' for i in range(4)}}
args = get_args()
logger = Logger("lenet5-VDO", fmt=fmt)
trainset = torchvision.datasets.MNIST(root='./data',
train=True,
download=True,
transform=transforms.ToTensor())
train_sampler = torch.utils.data.BatchSampler(
torch.utils.data.RandomSampler(trainset),
batch_size=args.batch_size,
drop_last=False)
trainloader = torch.utils.data.DataLoader(trainset,
batch_sampler=train_sampler,
num_workers=args.workers,
pin_memory=True)
testset = torchvision.datasets.MNIST(root='./data',
train=False,
download=True,
transform=transforms.ToTensor())
test_sampler = torch.utils.data.BatchSampler(
torch.utils.data.SequentialSampler(testset),
batch_size=args.batch_size,
drop_last=False)
testloader = torch.utils.data.DataLoader(testset,
batch_sampler=test_sampler,
num_workers=args.workers,
pin_memory=True)
net = LeNet5()
net = net.to(device=args.device, dtype=args.dtype)
if args.print_model:
logger.print(net)
criterion = metrics.SGVLB(net, len(trainset)).to(device=args.device,
dtype=args.dtype)
optimizer = optim.Adam(net.parameters(), lr=args.learning_rate)
epochs = args.epochs
lr_start = args.learning_rate
for epoch in trange(epochs): # loop over the dataset multiple times
t0 = time()
utils.adjust_learning_rate(
optimizer, metrics.lr_linear(epoch, 0, epochs, lr_start))
net.train()
training_loss = 0
accs = []
steps = 0
for i, (inputs, labels) in enumerate(tqdm(trainloader), 0):
steps += 1
inputs, labels = inputs.to(
device=args.device,
dtype=args.dtype), labels.to(device=args.device)
optimizer.zero_grad()
outputs = net(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
accs.append(metrics.logit2acc(
outputs.data,
labels)) # probably a bad way to calculate accuracy
training_loss += loss.item()
logger.add(epoch, tr_loss=training_loss / steps, tr_acc=np.mean(accs))
# Deterministic test
net.eval()
acc, nll = utils.evaluate(net,
testloader,
device=args.device,
num_ens=1)
logger.add(epoch, te_nll_det=nll, te_acc_det=acc)
# Stochastic test
net.train()
acc, nll = utils.evaluate(net,
testloader,
device=args.device,
num_ens=1)
logger.add(epoch, te_nll_stoch=nll, te_acc_stoch=acc)
# Test-time averaging
net.train()
acc, nll = utils.evaluate(net,
testloader,
device=args.device,
num_ens=20)
logger.add(epoch, te_nll_ens=nll, te_acc_ens=acc)
# Zero-mean
net.train()
net.dense1.set_flag('zero_mean', True)
acc, nll = utils.evaluate(net,
testloader,
device=args.device,
num_ens=1)
net.dense1.set_flag('zero_mean', False)
logger.add(epoch, te_nll_zero_mean=nll, te_acc_zero_mean=acc)
# Permuted sigmas
net.train()
net.dense1.set_flag('permute_sigma', True)
acc, nll = utils.evaluate(net,
testloader,
device=args.device,
num_ens=1)
net.dense1.set_flag('permute_sigma', False)
logger.add(epoch, te_nll_perm_sigma=nll, te_acc_perm_sigma=acc)
# Zero-mean test-time averaging
net.train()
net.dense1.set_flag('zero_mean', True)
acc, nll = utils.evaluate(net,
testloader,
device=args.device,
num_ens=20)
net.dense1.set_flag('zero_mean', False)
logger.add(epoch, te_nll_zero_mean_ens=nll, te_acc_zero_mean_ens=acc)
# Permuted sigmas test-time averaging
net.train()
net.dense1.set_flag('permute_sigma', True)
acc, nll = utils.evaluate(net,
testloader,
device=args.device,
num_ens=20)
net.dense1.set_flag('permute_sigma', False)
logger.add(epoch, te_nll_perm_sigma_ens=nll, te_acc_perm_sigma_ens=acc)
logger.add(epoch, time=time() - t0)
las = [
np.mean(net.conv1.log_alpha.data.cpu().numpy()),
np.mean(net.conv2.log_alpha.data.cpu().numpy()),
np.mean(net.dense1.log_alpha.data.cpu().numpy()),
np.mean(net.dense2.log_alpha.data.cpu().numpy())
]
logger.add(epoch, **{'la%d' % i: las[i] for i in range(4)})
logger.iter_info()
logger.save(silent=True)
torch.save(net.state_dict(), logger.checkpoint)
logger.save()
optimizer.step()
elbo.append(-loss.item())
cat_mean.append(categorical_mean.item())
kls.append(kl.item() if isinstance(kl, torch.Tensor) else kl)
accuracy.append(pred2acc(pred, y_train, args.batch_size))
t1 = time() - t0
elbo = np.mean(elbo)
cat_mean = np.mean(cat_mean)
kl = np.mean(kls)
accuracy = np.mean(accuracy)
logger.add(epoch,
kl=kl,
tr_elbo=elbo,
tr_acc=accuracy,
tr_ll=cat_mean,
tr_time=t1)
model.eval()
t_dvi = time()
test_acc_dvi = evaluate(model, test_loader, mode='dvi', args=args)
t_dvi = time() - t_dvi
if not args.no_mc:
t_mc = time()
test_acc_mcvi = evaluate(model,
test_loader,
mode='mcvi',
args=args)
t_mc = time() - t_mc
accs.append(metrics.logit2acc(log_outputs.data, labels))
training_loss += loss.cpu().data.numpy()[0]
# ELBO evaluation
net.train()
training_loss = 0
steps = 0
accs = []
for i, (inputs, labels) in enumerate(trainloader, 0):
steps += 1
inputs, labels = Variable(inputs.cuda(async=True)), Variable(labels.cuda(async=True))
for j in range(10):
outputs = net(inputs).detach()
training_loss += criterion(outputs, labels).cpu().data.numpy()[0]/10.0
accs.append(metrics.logit2acc(outputs.data, labels))
logger.add(epoch, kl=criterion.get_kl(), tr_loss=training_loss/steps, tr_acc=np.mean(accs))
# zero-mean ELBO evaluation
net.train()
net.set_flag('zero_mean', True)
training_loss = 0
steps = 0
accs = []
for i, (inputs, labels) in enumerate(trainloader, 0):
steps += 1
inputs, labels = Variable(inputs.cuda(async=True)), Variable(labels.cuda(async=True))
for j in range(10):
outputs = net(inputs).detach()
training_loss += criterion(net(inputs), labels).cpu().data.numpy()[0] / 10.0
accs.append(metrics.logit2acc(outputs.data, labels))
logger.add(epoch, zero_mean_tr_loss=training_loss / steps, zero_mean_tr_acc=np.mean(accs))
# ELBO evaluation
net.train()
training_loss = 0
steps = 0
accs = []
for i, (inputs, labels) in enumerate(trainloader, 0):
steps += 1
inputs, labels = Variable(inputs.cuda(async=True)), Variable(
labels.cuda(async=True))
for j in range(10):
outputs = net(inputs).detach()
training_loss += criterion(outputs,
labels).cpu().data.numpy()[0] / 10.0
accs.append(metrics.logit2acc(outputs.data, labels))
logger.add(epoch,
kl=criterion.get_kl(),
tr_loss=training_loss / steps,
tr_acc=np.mean(accs))
# zero-mean ELBO evaluation
net.train()
net.set_flag('zero_mean', True)
training_loss = 0
steps = 0
accs = []
for i, (inputs, labels) in enumerate(trainloader, 0):
steps += 1
inputs, labels = Variable(inputs.cuda(async=True)), Variable(
labels.cuda(async=True))
for j in range(10):
outputs = net(inputs).detach()
training_loss += criterion(net(inputs),
accs = []
steps = 0
for i, (inputs, labels) in enumerate(trainloader, 0):
steps += 1
inputs, labels = Variable(inputs.cuda(async=True)), Variable(labels.cuda(async=True))
optimizer.zero_grad()
outputs = net(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
accs.append(metrics.logit2acc(outputs.data, labels)) # probably a bad way to calculate accuracy
training_loss += loss.cpu().data.numpy()[0]
logger.add(epoch, tr_loss=training_loss/steps, tr_acc=np.mean(accs))
# Deterministic test
net.eval()
acc, nll = utils.evaluate(net, testloader, num_ens=1)
logger.add(epoch, te_nll_det=nll, te_acc_det=acc)
# Stochastic test
net.train()
acc, nll = utils.evaluate(net, testloader, num_ens=1)
logger.add(epoch, te_nll_stoch=nll, te_acc_stoch=acc)
# Test-time averaging
net.train()
acc, nll = utils.evaluate(net, testloader, num_ens=20)
logger.add(epoch, te_nll_ens=nll, te_acc_ens=acc)
