def eval_mnist_ensemble(ensemble, outlier=False):
for model in ensemble:
model.eval()
if outlier is True:
trainloader, testloader = datagen.load_notmnist()
else:
trainloader, testloader = datagen.load_mnist()
model_outputs = torch.zeros(len(ensemble), len(testloader.dataset), 10)
for i, (data, target) in enumerate(testloader):
data = data.cuda()
target = target.cuda()
outputs = []
for model in ensemble:
outputs.append(model(data))
outputs = torch.stack(outputs)
model_outputs[:, i * len(data):(i + 1) * len(data), :] = outputs
# Soft Voting (entropy in confidence)
probs_soft = F.softmax(model_outputs, dim=-1) # [ens, data, 10]
preds_soft = probs_soft.mean(0) # [data, 10]
entropy = entropy_fn(preds_soft.T.cpu().numpy()) # [data]
# Hard Voting (variance in predicted classed)
probs_hard = F.softmax(model_outputs, dim=-1) #[ens, data, 10]
preds_hard = probs_hard.var(0).cpu() # [data, 10]
variance = preds_hard.sum(1).numpy() # [data]
for model in ensemble:
model.train()
return entropy, variance
def eval_mnist_hypergan(hypergan, ens_size, s_dim, outlier=False):
hypergan.eval_()
if outlier is True:
trainloader, testloader = datagen.load_notmnist()
else:
trainloader, testloader = datagen.load_mnist()
model_outputs = torch.zeros(ens_size, len(testloader.dataset), 10)
for i, (data, target) in enumerate(testloader):
data = data.cuda()
target = target.cuda()
z = torch.randn(ens_size, s_dim).to(hypergan.device)
codes = hypergan.mixer(z)
params = hypergan.generator(codes)
outputs = []
for (layers) in zip(*params):
output = hypergan.eval_f(layers, data)
outputs.append(output)
outputs = torch.stack(outputs)
model_outputs[:, i * len(data):(i + 1) * len(data), :] = outputs
# Soft Voting (entropy in confidence)
probs_soft = F.softmax(model_outputs, dim=-1) # [ens, data, 10]
preds_soft = probs_soft.mean(0) # [data, 10]
entropy = entropy_fn(preds_soft.T.cpu().numpy()) # [data]
# Hard Voting (variance in predicted classed)
probs_hard = F.softmax(model_outputs, dim=-1) #[ens, data, 10]
preds_hard = probs_hard.var(0).cpu() # [data, 10]
variance = preds_hard.sum(1).numpy() # [data]
hypergan.train_()
return entropy, variance
def run_anomaly_notmnist(args, hypernet):
arch = get_network(args)
train, test = datagen.load_notmnist(args)
_vars, _stds, _ents = [], [], []
model = sample_model(hypernet, arch)
for n in [200]:
for idx, (data, target) in enumerate(test):
data, target = data.cuda(), target.cuda()
pred_labels = []
logits = []
for _ in range(n):
model = sample_model(hypernet, arch)
output = model(data)
logits.append(F.softmax(output, dim=1))
#logits.append(output)
probs = torch.stack(logits)
probs = probs.mean(0).float()
ent = torch.from_numpy(entropy(probs)).mean()
print(ent)
_ents.append(ent)
plot_empirical_cdf(args, _ents, n)
print('mean E: {}, max E: {}, min E:{}'.format(
torch.tensor(_ents).mean(),
torch.tensor(_ents).max(),
torch.tensor(_ents).min()))
def run_anomaly_notmnist_(args, hypernet):
arch = get_network(args)
train, test = datagen.load_notmnist(args)
_vars, _stds, _ents = [], [], []
model = sample_model(hypernet, arch)
for idx, (data, target) in enumerate(train):
data, target = data.cuda(), target.cuda()
pred_labels = []
logits = []
for _ in range(100):
model = sample_model(hypernet, arch)
output = model(data)
pred = output.data.max(1, keepdim=True)[1]
pred_labels.append(pred.view(pred.numel()))
p_labels = torch.stack(pred_labels).float().transpose(0, 1)
_vars.append(p_labels.var(1).mean())
_stds.append(p_labels.std(1).mean())
_ents.append(np.apply_along_axis(entropy, 1, p_labels))
plot_empirical_cdf(args, _ents)
print('mean var: {}, max var: {}, min var:{}, std: {}'.format(
torch.tensor(_vars).mean(),
torch.tensor(_vars).max(),
torch.tensor(_vars).min(),
torch.tensor(_stds).mean()))
def test_entropy(args, Z, names, arch, pop_size):
_, test_loader = datagen.load_notmnist(args)
criterion = nn.CrossEntropyLoss()
with torch.no_grad():
correct = 0.
test_loss = 0
preds = torch.zeros((10000, 10))
for idx, (data, target) in enumerate(test_loader):
data, target = data.cuda(), target.cuda()
outputs = []
for i in range(pop_size):
params = [Z[0][i], Z[1][i], Z[2][i]]
model = weights_to_clf(params, names, arch)
output = model(data)
outputs.append(F.softmax(output, dim=1))
pop_outputs = torch.stack(outputs)
pop_mean = pop_outputs.mean(0).view(100, -1)
preds[idx*100: (idx+1)*100] = pop_mean
#print (preds[idx*100: (idx+1)*100])
preds = preds.cpu().numpy()
ents = entropy(preds.T+1e-12)
return ents
def test_ent(args, Z, names, arch, pop_size, ds):
if ds == 'mnist':
_, test_loader = datagen.load_mnist(args)
if ds == 'notmnist':
_, test_loader = datagen.load_notmnist(args)
criterion = nn.CrossEntropyLoss()
with torch.no_grad():
correct = 0.
test_loss = 0
for data, target in test_loader:
data, target = data.cuda(), target.cuda()
outputs = []
for i in range(pop_size):
params = [Z[0][i], Z[1][i], Z[2][i]]
model = weights_to_clf(params, names, arch)
output = F.softmax(model(data))
outputs.append(output)
pop_outputs = torch.stack(outputs)
pop_outputs = pop_outputs.view(pop_size, 10000, 10)
pop_mean = pop_outputs.mean(0).view(10000, 10)
ent = entropy(pop_mean.cpu().numpy().T)
return ent