def test(nnName, dataName, CUDA_DEVICE, epsilon, temperature):
net1 = torch.load("../models/{}.pth".format(nnName))
optimizer1 = optim.SGD(net1.parameters(), lr = 0, momentum = 0)
net1.cuda(CUDA_DEVICE)
if dataName != "Uniform" and dataName != "Gaussian":
testsetout = torchvision.datasets.ImageFolder("../data/{}".format(dataName), transform=transform)
testloaderOut = torch.utils.data.DataLoader(testsetout, batch_size=1,
shuffle=False, num_workers=2)
if nnName == "densenet10" or nnName == "wideresnet10":
testset = torchvision.datasets.CIFAR10(root='../data', train=False, download=True, transform=transform)
testloaderIn = torch.utils.data.DataLoader(testset, batch_size=1,
shuffle=False, num_workers=2)
if nnName == "densenet100" or nnName == "wideresnet100":
testset = torchvision.datasets.CIFAR100(root='../data', train=False, download=True, transform=transform)
testloaderIn = torch.utils.data.DataLoader(testset, batch_size=1,
shuffle=False, num_workers=2)
if dataName == "Gaussian":
d.testGaussian(net1, criterion, CUDA_DEVICE, testloaderIn, testloaderIn, nnName, dataName, epsilon, temperature)
m.metric(nnName, dataName)
elif dataName == "Uniform":
d.testUni(net1, criterion, CUDA_DEVICE, testloaderIn, testloaderIn, nnName, dataName, epsilon, temperature)
m.metric(nnName, dataName)
else:
d.testData(net1, criterion, CUDA_DEVICE, testloaderIn, testloaderOut, nnName, dataName, epsilon, temperature)
m.metric(nnName, dataName)
def test(nnName, dataName, epsilon, temperature):
"""
:param nnName: in-distribution
:param dataName: out-of-distribution
:param epsilon: noiseMagnitude
:param temperature: scaling
"""
print("--start testing!--")
net1 = cifar10vgg(train=False).model
if nnName == "densenet10":
(x_train, y_train), (x_test,
y_test) = tf.keras.datasets.cifar10.load_data()
# testloaderIn=x_train[:10000]
testloaderIn = x_train[:10000]
if dataName == "Imagenet_crop":
# (x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar100.load_data()
testloaderOut = load_images_from_folder("./Imagenet/test")
if dataName == "CIFAR-100":
(_, _), (testloaderOut, _) = tf.keras.datasets.cifar100.load_data()
if dataName == "Gaussian":
testloaderOut = np.random.standard_normal(
size=testloaderIn.shape) + 0.5
if dataName == "Uniform":
testloaderOut = np.random.uniform(0, 1, size=testloaderIn.shape)
testloaderIn, testloaderOut = normalize(testloaderIn, testloaderOut)
testloaderIn = (testloaderIn, y_train[:10000])
d.testData(net1, testloaderIn, testloaderOut, nnName, dataName, epsilon,
temperature)
m.metric(nnName, dataName, temperature, epsilon)
def test(nnName, dataName, CUDA_DEVICE, epsilon, temperature):
model = DenseNetBC_50_12()
model.load_state_dict(torch.load("../models/{}.pth".format(nnName)))
# checkpoint = torch.load("../checkpoints_healthy/{}.pth.tar".format(nnName))
# model.load_state_dict(checkpoint['state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer'])
optimizer1 = optim.SGD(model.parameters(), lr=0, momentum=0)
for i, (name, module) in enumerate(model._modules.items()):
module = recursion_change_bn(model)
model.cuda(CUDA_DEVICE)
transform_test = transforms.Compose(
[transforms.Resize((512, 512)),
transforms.ToTensor()])
testsetout = torchvision.datasets.ImageFolder(
"/home/yoon/jyk416/odin-pytorch/data/{}".format(dataName),
transform=transform_test)
testloaderOut = torch.utils.data.DataLoader(testsetout,
batch_size=1,
shuffle=False,
num_workers=2)
# if dataName != "Uniform" and dataName != "Gaussian":
# testsetout = torchvision.datasets.ImageFolder("../data/{}".format(dataName), transform=transform)
# testloaderOut = torch.utils.data.DataLoader(testsetout, batch_size=1,
# shuffle=False, num_workers=2)
train_test_dir = '/home/yoon/jyk416/odin-pytorch/data/train3'
if nnName == "model104":
testset = torchvision.datasets.ImageFolder(train_test_dir,
transform=transform_test)
testloaderIn = torch.utils.data.DataLoader(testset,
batch_size=1,
shuffle=True,
num_workers=2)
# if nnName == "densenet10" or nnName == "wideresnet10":
# testset = torchvision.datasets.CIFAR10(root='../data', train=False, download=True, transform=transform)
# testloaderIn = torch.utils.data.DataLoader(testset, batch_size=1,
# shuffle=False, num_workers=2)
# if nnName == "densenet100" or nnName == "wideresnet100":
# testset = torchvision.datasets.CIFAR100(root='../data', train=False, download=True, transform=transform)
# testloaderIn = torch.utils.data.DataLoader(testset, batch_size=1,
# shuffle=False, num_workers=2)
# if dataName == "Gaussian":
# d.testGaussian(net1, criterion, CUDA_DEVICE, testloaderIn, testloaderIn, nnName, dataName, epsilon, temperature)
# m.metric(nnName, dataName)
#
# elif dataName == "Uniform":
# d.testUni(net1, criterion, CUDA_DEVICE, testloaderIn, testloaderIn, nnName, dataName, epsilon, temperature)
# m.metric(nnName, dataName)
# else:
# d.testData(net1, criterion, CUDA_DEVICE, testloaderIn, testloaderOut, nnName, dataName, epsilon, temperature)
# m.metric(nnName, dataName)
d.testData(model, criterion, CUDA_DEVICE, testloaderIn, testloaderOut,
nnName, epsilon, temperature)
m.metric(nnName, dataName)
def val(nnName, dataName, CUDA_DEVICE, temperature, val_min, val_max, val_num):
net1 = torch.load("../models/{}.pth".format(nnName))
optimizer1 = optim.SGD(net1.parameters(), lr=0, momentum=0)
net1.cuda(CUDA_DEVICE)
testsetout = torchvision.datasets.ImageFolder(
"../data/{}".format(dataName), transform=transform)
rand_sampler = torch.utils.data.sampler.RandomSampler(testsetout)
testloaderOut = torch.utils.data.DataLoader(testsetout,
sampler=rand_sampler,
batch_size=1,
shuffle=False,
num_workers=2)
if nnName == "densenet10":
testset = torchvision.datasets.CIFAR10(root='../data',
train=False,
download=True,
transform=transform)
testloaderIn = torch.utils.data.DataLoader(testset,
batch_size=1,
shuffle=False,
num_workers=2)
if nnName == "densenet100":
testset = torchvision.datasets.CIFAR100(root='../data',
train=False,
download=True,
transform=transform)
testloaderIn = torch.utils.data.DataLoader(testset,
batch_size=1,
shuffle=False,
num_workers=2)
# validate epsilon by picking argmax of tpr95
eps_fpr_pairs = {}
print("Validating with", val_num, "parameter samples in range", val_min,
"to", val_max)
round_num = 1
for epsilon in np.linspace(val_min, val_max, num=val_num):
print("Round", round_num, "of", val_num, ": eps =", epsilon)
round_num += 1
d.testData(net1, criterion, CUDA_DEVICE, testloaderIn, testloaderOut,
nnName, dataName, epsilon, temperature)
fpr = m.val_metric(nnName, dataName)
eps_fpr_pairs[epsilon] = fpr
best_pair = None
for (eps, fpr) in eps_fpr_pairs.items():
if best_pair == None or fpr < best_pair[1]:
best_pair = (eps, fpr)
print("Finished validation")
print("Best epsilon found:", best_pair[0], "at fpr", best_pair[1])
print("All eps/fpr pairs tested:", eps_fpr_pairs)
def test(in_dataset, out_dataset, wide, epsilon, temperature):
testsetout = torchvision.datasets.ImageFolder(os.path.expanduser(
"./data/{}".format(out_dataset)),
transform=transform)
testloaderOut = torch.utils.data.DataLoader(testsetout,
batch_size=100,
shuffle=False,
num_workers=2)
if in_dataset == "cifar100":
testset = torchvision.datasets.CIFAR100(root='./data',
train=False,
download=True,
transform=transform)
testloaderIn = torch.utils.data.DataLoader(testset,
batch_size=100,
shuffle=False,
num_workers=2)
elif in_dataset == "cifar10":
testset = torchvision.datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=transform)
testloaderIn = torch.utils.data.DataLoader(testset,
batch_size=100,
shuffle=False,
num_workers=2)
for fold in range(1, 6):
print(f"Processing fold {fold}")
nclasses = int(in_dataset[5:])
if wide:
net = models.WideResNet(int(nclasses * 4 / 5))
ck = torch.load(
f"./checkpoints/{in_dataset}_fold_{fold}_wide_checkpoint/model_best.pth.tar"
)
else:
net = models.DenseNet(int(nclasses * 4 / 5))
ck = torch.load(
f"./checkpoints/{in_dataset}_fold_{fold}_dense_checkpoint/model_best.pth.tar"
)
net.load_state_dict(ck['state_dict'])
net.cuda()
net.eval()
d.testData(net, criterion, testloaderIn, testloaderOut, in_dataset,
out_dataset, epsilon, temperature, fold)
m.test(in_dataset, out_dataset, plot=True)
def test(nnName, dataName, CUDA_DEVICE, epsilon, temperature):
use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
net1 = vgg1.VGG('VGG16').to(device)
# Test set: Average loss: 0.0015, Accuracy: 9337 / 10000(93 %)
if use_cuda:
net1 = torch.nn.DataParallel(net1)
cudnn.benchmark = True
checkpoint = torch.load("../../model/{}.pth".format(nnName))
net1.load_state_dict(checkpoint['net'])
net1.eval()
if dataName != "Uniform" and dataName != "Gaussian":
testsetout = torchvision.datasets.ImageFolder(
"../../data/{}".format(dataName), transform=transform)
testloaderOut = torch.utils.data.DataLoader(testsetout,
batch_size=1,
shuffle=False,
num_workers=2)
# if nnName == "densenet10" or nnName == "wideresnet10":
testset = torchvision.datasets.CIFAR10(root='../../data',
train=False,
download=True,
transform=transform)
testloaderIn = torch.utils.data.DataLoader(testset,
batch_size=1,
shuffle=False,
num_workers=2)
# if nnName == "densenet100" or nnName == "wideresnet100":
# testset = torchvision.datasets.CIFAR100(root='../../data', train=False, download=True, transform=transform)
# testloaderIn = torch.utils.data.DataLoader(testset, batch_size=1,
# shuffle=False, num_workers=2)
if dataName == "Gaussian":
d.testGaussian(net1, criterion, CUDA_DEVICE, testloaderIn,
testloaderIn, nnName, dataName, epsilon, temperature)
m.metric(nnName, dataName)
elif dataName == "Uniform":
d.testUni(net1, criterion, CUDA_DEVICE, testloaderIn, testloaderIn,
nnName, dataName, epsilon, temperature)
m.metric(nnName, dataName)
else:
d.testData(net1, criterion, CUDA_DEVICE, testloaderIn, testloaderOut,
nnName, dataName, epsilon, temperature)
m.metric(nnName, dataName)
def test(nnName, dataName, CUDA_DEVICE, epsilon, temperature):
net1 = torch.load("../models/{}.pth".format(nnName))
optimizer1 = optim.SGD(net1.parameters(), lr = 0, momentum = 0)
net1.cuda(CUDA_DEVICE)
if dataName != "Uniform" and dataName != "Gaussian":
if dataName == "SVHN":
testsetout = svhn.SVHN("../data/SVHN", split='test', transform=transform, download=True)
elif dataName in ["HFlip","VFlip"]:
testsetout = torchvision.datasets.CIFAR10('../data', train=False, download=True,
transform=Flip[dataName])
elif dataName == "CelebA":
testsetout = torchvision.datasets.ImageFolder(
"../data/{}".format(dataName),
transform=transforms.Compose([transforms.CenterCrop(178), Resize(32), transform]))
else:
testsetout = torchvision.datasets.ImageFolder("../data/{}".format(dataName),
transform=transform)
testloaderOut = torch.utils.data.DataLoader(testsetout, batch_size=1,
shuffle=False, num_workers=2)
if nnName == "densenet10" or nnName == "wideresnet10":
testset = torchvision.datasets.CIFAR10(root='../data', train=False, download=True, transform=transform)
testloaderIn = torch.utils.data.DataLoader(testset, batch_size=1,
shuffle=False, num_workers=2)
if nnName == "densenet100" or nnName == "wideresnet100":
testset = torchvision.datasets.CIFAR100(root='../data', train=False, download=True, transform=transform)
testloaderIn = torch.utils.data.DataLoader(testset, batch_size=1,
shuffle=False, num_workers=2)
if dataName == "Gaussian":
d.testGaussian(net1, criterion, CUDA_DEVICE, testloaderIn, testloaderIn, nnName, dataName, epsilon, temperature)
m.metric(nnName, dataName)
elif dataName == "Uniform":
d.testUni(net1, criterion, CUDA_DEVICE, testloaderIn, testloaderIn, nnName, dataName, epsilon, temperature)
m.metric(nnName, dataName)
else:
d.testData(net1, criterion, CUDA_DEVICE, testloaderIn, testloaderOut, nnName, dataName, epsilon, temperature)
m.metric(nnName, dataName)
def detect_ood_odin(self,
gpu,
nnName="model",
epsilon=0.0014,
temperature=1000):
net1 = self.model
testloaderIn = torch.utils.data.DataLoader(self.in_distribution,
batch_size=1,
shuffle=False,
num_workers=2)
testloadersOut = []
for dataset in self.out_of_distribution:
testloadersOut.append(
torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=2))
net1.cuda(gpu)
d.testData(net1, torch.nn.CrossEntropyLoss(), gpu, testloaderIn,
testloadersOut, nnName, self.data_lables[0], epsilon,
temperature)
m.metric(nnName, self.data_lables[0])