def evaluate(net, dataloader, num_ens=1):
"""Calculate ensemble accuracy and NLL"""
accs = []
nlls = []
for i, (inputs, labels) in enumerate(dataloader):
inputs, labels = Variable(inputs.cuda(async=True)), Variable(labels.cuda(async=True))
outputs = torch.zeros(inputs.shape[0], net.num_classes, num_ens).cuda()
for j in range(num_ens):
outputs[:, :, j] = F.log_softmax(net(inputs), dim=1).data
accs.append(metrics.logit2acc(logmeanexp(outputs, dim=2), labels))
nlls.append(F.nll_loss(Variable(logmeanexp(outputs, dim=2)), labels, size_average=False).data.cpu().numpy())
return np.mean(accs), np.sum(nlls)
def evaluate(net, dataloader, device, num_ens=1):
"""Calculate ensemble accuracy and NLL"""
accs = []
nlls = []
for i, (inputs, labels) in enumerate(dataloader):
inputs, labels = inputs.to(device=device), labels.to(device=device)
outputs = torch.zeros(inputs.shape[0], net.num_classes, num_ens).to(device=device)
for j in range(num_ens):
outputs[:, :, j] = F.log_softmax(net(inputs), dim=1).data
accs.append(metrics.logit2acc(logmeanexp(outputs, dim=2), labels))
nlls.append(F.nll_loss(Variable(logmeanexp(outputs, dim=2)), labels, reduction='sum').data.cpu().numpy())
return np.mean(accs), np.sum(nlls)
def evaluate(net, dataloader, num_ens=1):
"""Calculate ensemble accuracy and NLL"""
accs = []
nlls = []
for i, (inputs, labels) in enumerate(dataloader):
inputs, labels = Variable(inputs.cuda(async=True)), Variable(
labels.cuda(async=True))
outputs = torch.zeros(inputs.shape[0], net.num_classes, num_ens).cuda()
for j in range(num_ens):
outputs[:, :, j] = F.log_softmax(net(inputs), dim=1).data
accs.append(metrics.logit2acc(logmeanexp(outputs, dim=2), labels))
nlls.append(
F.nll_loss(Variable(logmeanexp(outputs, dim=2)),
labels,
size_average=False).data.cpu().numpy())
return np.mean(accs), np.sum(nlls)
for i, (inputs, labels) in enumerate(trainloader, 0):
steps += 1
inputs, labels = Variable(inputs.cuda(async=True)), Variable(labels.cuda(async=True))
optimizer.zero_grad()
outputs = Variable(torch.zeros(inputs.shape[0], net.num_classes, flags.train_ens).cuda())
for j in range(flags.train_ens):
outputs[:, :, j] = F.log_softmax(net(inputs), dim=1)
log_outputs = utils.logmeanexp(outputs, dim=2)
loss = criterion(log_outputs, labels)
loss.backward()
optimizer.step()
accs.append(metrics.logit2acc(log_outputs.data, labels))
training_loss += loss.cpu().data.numpy()[0]
logger.add(epoch, tr_loss=training_loss/steps, tr_acc=np.mean(accs))
# Ens 100 test
net.train()
acc, nll = utils.evaluate(net, testloader, num_ens=100)
logger.add(epoch, te_nll_ens100=nll, te_acc_ens100=acc)
# Stochastic test
net.train()
acc, nll = utils.evaluate(net, testloader, num_ens=1)
logger.add(epoch, te_nll_stoch=nll, te_acc_stoch=acc)
# Test-time averaging
utils.adjust_learning_rate(optimizer, metrics.lr_linear(epoch, 0, epochs, lr_start))
net.train()
training_loss = 0
accs = []
steps = 0
for i, (inputs, labels) in enumerate(trainloader, 0):
steps += 1
inputs, labels = Variable(inputs.cuda(async=True)), Variable(labels.cuda(async=True))
optimizer.zero_grad()
outputs = net(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
accs.append(metrics.logit2acc(outputs.data, labels)) # probably a bad way to calculate accuracy
training_loss += loss.cpu().data.numpy()[0]
logger.add(epoch, tr_loss=training_loss/steps, tr_acc=np.mean(accs))
# Deterministic test
net.eval()
acc, nll = utils.evaluate(net, testloader, num_ens=1)
logger.add(epoch, te_nll_det=nll, te_acc_det=acc)
# Stochastic test
net.train()
acc, nll = utils.evaluate(net, testloader, num_ens=1)
logger.add(epoch, te_nll_stoch=nll, te_acc_stoch=acc)
# Test-time averaging
for i, (inputs, labels) in enumerate(trainloader, 0):
steps += 1
inputs, labels = Variable(inputs.cuda(async=True)), Variable(labels.cuda(async=True))
optimizer.zero_grad()
outputs = Variable(torch.zeros(inputs.shape[0], net.num_classes, flags.train_ens).cuda())
for j in range(flags.train_ens):
outputs[:, :, j] = F.log_softmax(net(inputs), dim=1)
log_outputs = utils.logmeanexp(outputs, dim=2)
loss = criterion(log_outputs, labels)
loss.backward()
optimizer.step()
accs.append(metrics.logit2acc(log_outputs.data, labels))
training_loss += loss.cpu().data.numpy()[0]
logger.add(epoch, tr_loss=training_loss/steps, tr_acc=np.mean(accs))
# Ens 100 test
net.train()
acc, nll = utils.evaluate(net, testloader, num_ens=100)
logger.add(epoch, te_nll_ens100=nll, te_acc_ens100=acc)
# Stochastic test
net.train()
acc, nll = utils.evaluate(net, testloader, num_ens=1)
logger.add(epoch, te_nll_stoch=nll, te_acc_stoch=acc)
# Test-time averaging
def main():
fmt = {
'tr_loss': '3.1e',
'tr_acc': '.4f',
'te_acc_det': '.4f',
'te_acc_stoch': '.4f',
'te_acc_ens': '.4f',
'te_acc_perm_sigma': '.4f',
'te_acc_zero_mean': '.4f',
'te_acc_perm_sigma_ens': '.4f',
'te_acc_zero_mean_ens': '.4f',
'te_nll_det': '.4f',
'te_nll_stoch': '.4f',
'te_nll_ens': '.4f',
'te_nll_perm_sigma': '.4f',
'te_nll_zero_mean': '.4f',
'te_nll_perm_sigma_ens': '.4f',
'te_nll_zero_mean_ens': '.4f',
'time': '.3f'
}
fmt = {**fmt, **{'la%d' % i: '.4f' for i in range(4)}}
args = get_args()
logger = Logger("lenet5-VDO", fmt=fmt)
trainset = torchvision.datasets.MNIST(root='./data',
train=True,
download=True,
transform=transforms.ToTensor())
train_sampler = torch.utils.data.BatchSampler(
torch.utils.data.RandomSampler(trainset),
batch_size=args.batch_size,
drop_last=False)
trainloader = torch.utils.data.DataLoader(trainset,
batch_sampler=train_sampler,
num_workers=args.workers,
pin_memory=True)
testset = torchvision.datasets.MNIST(root='./data',
train=False,
download=True,
transform=transforms.ToTensor())
test_sampler = torch.utils.data.BatchSampler(
torch.utils.data.SequentialSampler(testset),
batch_size=args.batch_size,
drop_last=False)
testloader = torch.utils.data.DataLoader(testset,
batch_sampler=test_sampler,
num_workers=args.workers,
pin_memory=True)
net = LeNet5()
net = net.to(device=args.device, dtype=args.dtype)
if args.print_model:
logger.print(net)
criterion = metrics.SGVLB(net, len(trainset)).to(device=args.device,
dtype=args.dtype)
optimizer = optim.Adam(net.parameters(), lr=args.learning_rate)
epochs = args.epochs
lr_start = args.learning_rate
for epoch in trange(epochs): # loop over the dataset multiple times
t0 = time()
utils.adjust_learning_rate(
optimizer, metrics.lr_linear(epoch, 0, epochs, lr_start))
net.train()
training_loss = 0
accs = []
steps = 0
for i, (inputs, labels) in enumerate(tqdm(trainloader), 0):
steps += 1
inputs, labels = inputs.to(
device=args.device,
dtype=args.dtype), labels.to(device=args.device)
optimizer.zero_grad()
outputs = net(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
accs.append(metrics.logit2acc(
outputs.data,
labels)) # probably a bad way to calculate accuracy
training_loss += loss.item()
logger.add(epoch, tr_loss=training_loss / steps, tr_acc=np.mean(accs))
# Deterministic test
net.eval()
acc, nll = utils.evaluate(net,
testloader,
device=args.device,
num_ens=1)
logger.add(epoch, te_nll_det=nll, te_acc_det=acc)
# Stochastic test
net.train()
acc, nll = utils.evaluate(net,
testloader,
device=args.device,
num_ens=1)
logger.add(epoch, te_nll_stoch=nll, te_acc_stoch=acc)
# Test-time averaging
net.train()
acc, nll = utils.evaluate(net,
testloader,
device=args.device,
num_ens=20)
logger.add(epoch, te_nll_ens=nll, te_acc_ens=acc)
# Zero-mean
net.train()
net.dense1.set_flag('zero_mean', True)
acc, nll = utils.evaluate(net,
testloader,
device=args.device,
num_ens=1)
net.dense1.set_flag('zero_mean', False)
logger.add(epoch, te_nll_zero_mean=nll, te_acc_zero_mean=acc)
# Permuted sigmas
net.train()
net.dense1.set_flag('permute_sigma', True)
acc, nll = utils.evaluate(net,
testloader,
device=args.device,
num_ens=1)
net.dense1.set_flag('permute_sigma', False)
logger.add(epoch, te_nll_perm_sigma=nll, te_acc_perm_sigma=acc)
# Zero-mean test-time averaging
net.train()
net.dense1.set_flag('zero_mean', True)
acc, nll = utils.evaluate(net,
testloader,
device=args.device,
num_ens=20)
net.dense1.set_flag('zero_mean', False)
logger.add(epoch, te_nll_zero_mean_ens=nll, te_acc_zero_mean_ens=acc)
# Permuted sigmas test-time averaging
net.train()
net.dense1.set_flag('permute_sigma', True)
acc, nll = utils.evaluate(net,
testloader,
device=args.device,
num_ens=20)
net.dense1.set_flag('permute_sigma', False)
logger.add(epoch, te_nll_perm_sigma_ens=nll, te_acc_perm_sigma_ens=acc)
logger.add(epoch, time=time() - t0)
las = [
np.mean(net.conv1.log_alpha.data.cpu().numpy()),
np.mean(net.conv2.log_alpha.data.cpu().numpy()),
np.mean(net.dense1.log_alpha.data.cpu().numpy()),
np.mean(net.dense2.log_alpha.data.cpu().numpy())
]
logger.add(epoch, **{'la%d' % i: las[i] for i in range(4)})
logger.iter_info()
logger.save(silent=True)
torch.save(net.state_dict(), logger.checkpoint)
logger.save()