def test(self, data, adversarial, test_noise=args.test_noise):
"""
Computes Mahalanobis scores for test set in_data, out_data
Args:
- data: name of out dataset
- adversarial: boolean flag for if this is an adversarial input
- ood boolean flag for if this is not the in-dataset
- test_noise: constant noise to add to test data to help separation
"""
adversarial = data in ADVERSARIAL # an adversarial out
positive = data == self.in_data # true label in
if positive:
test_loader = self.in_test_loader
elif not adversarial:
# if not adversarial, using torch.DataLoader, so just load once
test_loader = data_loader.getNonTargetDataSet(
data, self.batch_size, args.data_path)
if args.verbose:
print(">> Testing on dataset ", data)
Mahalanobis_scores = np.array([])
for i in range(self.num_layers):
if args.verbose:
print(">> Layer ", i)
# if adversarial, using a list, and for some reason, need to load this continuously
if adversarial:
test_loader = data_loader.getAdversarialDataSet(
data, args.model, self.in_data, self.batch_size)
layer_scores = self._get_Mahalanobis_score(data, test_loader, i,
test_noise)
layer_scores = np.expand_dims(layer_scores, axis=1) #N, -> Nx1
Mahalanobis_scores = np.hstack(
(Mahalanobis_scores,
layer_scores)) if Mahalanobis_scores.size else layer_scores
# print(Mahalanobis_scores.shape)
# save results
Mahalanobis_labels = np.ones(
Mahalanobis_scores.shape[0]) if positive else np.zeros(
Mahalanobis_scores.shape[0])
Mahalanobis_labels = np.expand_dims(Mahalanobis_labels,
axis=1) #N, -> Nx1
Mahalanobis_data = np.hstack((Mahalanobis_scores, Mahalanobis_labels))
file_name = os.path.join(
self.save_path,
'Mahalanobis_%s_%s_%s.npy' % (str(test_noise), self.in_data, data))
if args.verbose:
print(">> Writing cumulative results to ", file_name)
np.save(file_name, Mahalanobis_data)
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
if args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
################################ edits
print("Calculate SVD before getting sample mean and variance")
# svd_result= lib_generation.get_pca(model, args.num_classes, feature_list, train_loader)
# lib_generation.get_pca_incremental(model, args.num_classes, feature_list, train_loader,args)
svd_result = None
print('get sample mean and covariance')
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader, svd_result, args)
################################ edits_end_sample_generator
print('get Mahalanobis scores')
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print('Noise: ' + str(magnitude))
for i in range(num_output):
M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \
True, args.net_type, sample_mean, precision, i, magnitude,svd_result,args)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
print('Out-distribution: ' + out_dist)
for i in range(num_output):
M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \
False, args.net_type, sample_mean, precision, i, magnitude,svd_result,args)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_in)
file_name = os.path.join(
args.outf, 'Mahalanobis_%s_%s_%s.npy' %
(str(magnitude), args.dataset, out_dist))
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
if args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize',] #'lsun_resize']
else:
out_dist_list = ['svhn', 'imagenet_resize',] #'lsun_resize']
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(torch.load(pre_trained_net, map_location = "cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net, map_location = "cuda:" + str(args.gpu))
in_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize([0.5071, 0.4867, 0.4408], [0.2675, 0.2565, 0.2761]),])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(torch.load(pre_trained_net, map_location = "cuda:" + str(args.gpu)))
in_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot)
# measure the performance
M_list = [0, 0.0005, 0.001, 0.0014, 0.002, 0.0024, 0.005, 0.01, 0.05, 0.1, 0.2]
T_list = [1, 10, 100, 1000]
base_line_list = []
ODIN_best_tnr = [0, 0, 0]
ODIN_best_results = [0 , 0, 0]
ODIN_best_temperature = [-1, -1, -1]
ODIN_best_magnitude = [-1, -1, -1]
for T in T_list:
for m in M_list:
magnitude = m
temperature = T
lib_generation.get_posterior(model, args.net_type, test_loader, magnitude, temperature, args.outf, True)
out_count = 0
print('Temperature: ' + str(temperature) + ' / noise: ' + str(magnitude))
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(out_dist, args.batch_size, in_transform, args.dataroot)
print('Out-distribution: ' + out_dist)
lib_generation.get_posterior(model, args.net_type, out_test_loader, magnitude, temperature, args.outf, False)
if temperature == 1 and magnitude == 0:
test_results = callog.metric(args.outf, ['PoT'])
base_line_list.append(test_results)
else:
val_results = callog.metric(args.outf, ['PoV'])
if ODIN_best_tnr[out_count] < val_results['PoV']['TNR']:
ODIN_best_tnr[out_count] = val_results['PoV']['TNR']
ODIN_best_results[out_count] = callog.metric(args.outf, ['PoT'])
ODIN_best_temperature[out_count] = temperature
ODIN_best_magnitude[out_count] = magnitude
out_count += 1
# print the results
mtypes = ['TNR', 'AUROC', 'DTACC', 'AUIN', 'AUOUT']
print('Baseline method: in_distribution: ' + args.dataset + '==========')
count_out = 0
for results in base_line_list:
print('out_distribution: '+ out_dist_list[count_out])
for mtype in mtypes:
print(' {mtype:6s}'.format(mtype=mtype), end='')
print('\n{val:6.2f}'.format(val=100.*results['PoT']['TNR']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUROC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['DTACC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUIN']), end='')
print(' {val:6.2f}\n'.format(val=100.*results['PoT']['AUOUT']), end='')
print('')
count_out += 1
print('ODIN method: in_distribution: ' + args.dataset + '==========')
count_out = 0
for results in ODIN_best_results:
print('out_distribution: '+ out_dist_list[count_out])
for mtype in mtypes:
print(' {mtype:6s}'.format(mtype=mtype), end='')
print('\n{val:6.2f}'.format(val=100.*results['PoT']['TNR']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUROC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['DTACC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUIN']), end='')
print(' {val:6.2f}\n'.format(val=100.*results['PoT']['AUOUT']), end='')
print('temperature: ' + str(ODIN_best_temperature[count_out]))
print('magnitude: '+ str(ODIN_best_magnitude[count_out]))
print('')
count_out += 1
def main():
# set the path to pre-trained model and output
pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth"
args.outf = args.outf + args.net_type + "_" + args.dataset + "/"
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == "cifar100":
args.num_classes = 100
if args.dataset == "svhn":
out_dist_list = ["cifar10", "imagenet_resize", "lsun_resize"]
else:
out_dist_list = ["svhn", "imagenet_resize", "lsun_resize"]
# load networks
if args.net_type == "densenet":
if args.dataset == "svhn":
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
(125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0),
),
])
elif args.net_type == "resnet":
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print("load model: " + args.net_type)
# load dataset
print("load target data: ", args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print("get sample mean and covariance")
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader)
print("get Mahalanobis scores")
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print("Noise: " + str(magnitude))
for i in range(num_output):
M_in = lib_generation.get_Mahalanobis_score(
model,
test_loader,
args.num_classes,
args.outf,
True,
args.net_type,
sample_mean,
precision,
i,
magnitude,
)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
print("Out-distribution: " + out_dist)
for i in range(num_output):
M_out = lib_generation.get_Mahalanobis_score(
model,
out_test_loader,
args.num_classes,
args.outf,
False,
args.net_type,
sample_mean,
precision,
i,
magnitude,
)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
(
Mahalanobis_data,
Mahalanobis_labels,
) = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_in)
file_name = os.path.join(
args.outf,
"Mahalanobis_%s_%s_%s.npy" %
(str(magnitude), args.dataset, out_dist),
)
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
def main():
if os.path.isdir('feature_lists') == False:
os.mkdir('feature_lists')
if args.dataset == 'cifar100':
args.num_classes = 100
else:
args.num_classes = 10
# load networks
pre_trained_net = args.net_type + '_' + args.dataset + '.pth'
pre_trained_net = os.path.join('pre_trained', pre_trained_net)
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(0)))
else:
model = torch.load(pre_trained_net, map_location="cpu")
for i, (name, module) in enumerate(model._modules.items()):
module = recursion_change_bn(model)
for m in model.modules():
if 'Conv' in str(type(m)):
setattr(m, 'padding_mode', 'zeros')
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(0)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
if args.validation_src == 'FGSM':
if args.dataset == 'svhn':
out_dist_list = [
'cifar10', 'imagenet_resize', 'lsun_resize', 'FGSM'
]
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize', 'FGSM']
else:
if args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
print('load model: ' + args.net_type)
model.to(device)
model.eval()
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extraction
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
correct, total = 0, 0
num_output = len(feature_list)
num_sample_per_class = np.empty(args.num_classes)
num_sample_per_class.fill(0)
list_features = []
list_features_test = []
list_features_out = []
for i in range(num_output):
temp_list = []
list_features_test.append(0)
list_features_out.append(0)
for j in range(args.num_classes):
temp_list.append(0)
list_features.append(temp_list)
for data, target in train_loader:
total += data.size(0)
data = data.cuda()
data = Variable(data, volatile=True)
output, out_features = model.feature_list(data)
# get hidden features
for i in range(num_output):
out_features[i] = out_features[i].view(out_features[i].size(0),
out_features[i].size(1), -1)
out_features[i] = torch.mean(out_features[i].data, 2)
# compute the accuracy
pred = output.data.max(1)[1]
equal_flag = pred.eq(target.cuda()).cpu()
correct += equal_flag.sum()
# construct the sample matrix
for i in range(data.size(0)):
label = target[i]
if num_sample_per_class[label] == 0:
out_count = 0
for out in out_features:
list_features[out_count][label] = out[i].view(1, -1)
out_count += 1
else:
out_count = 0
for out in out_features:
list_features[out_count][label] = torch.cat(
(list_features[out_count][label], out[i].view(1, -1)),
0)
out_count += 1
num_sample_per_class[label] += 1
sample_class_mean = []
out_count = 0
for num_feature in feature_list:
temp_list = torch.Tensor(args.num_classes, int(num_feature)).cuda()
for j in range(args.num_classes):
temp_list[j] = torch.mean(list_features[out_count][j], 0)
sample_class_mean.append(temp_list)
out_count += 1
A = []
A_inv = []
log_abs_det_A_inv = []
for k in range(num_output):
X = 0
for i in range(args.num_classes):
if i == 0:
X = list_features[k][i] - sample_class_mean[k][i]
else:
X = torch.cat(
(X, list_features[k][i] - sample_class_mean[k][i]), 0)
# find inverse
u, s, vh = np.linalg.svd((X.cpu().numpy()) / np.sqrt(X.shape[0]),
full_matrices=False)
covariance_real = np.cov(X.cpu().numpy().T)
valid_indx = s > 1e-5
if (valid_indx.sum() % 2 > 0):
valid_indx[valid_indx.sum() - 1] = False
covriance_cal = np.matmul(
np.matmul(vh[valid_indx, :].transpose(),
np.diag(s[valid_indx]**2)), vh[valid_indx, :])
A_temp = np.matmul(vh[valid_indx, :].transpose(),
np.diag(s[valid_indx]))
A.append(A_temp)
covriance_cal2 = np.matmul(A_temp, A_temp.transpose())
s_inv = 1 / s[valid_indx]
A_inv_temp = np.matmul(np.diag(s_inv), vh[valid_indx, :])
A_inv.append(A_inv_temp)
log_abs_det_A_inv_temp = np.sum(np.log(np.abs(s_inv)))
log_abs_det_A_inv.append(log_abs_det_A_inv_temp)
print('\n Training Accuracy:({:.2f}%)\n'.format(100.0 * int(correct) /
int(total)))
num_sample_per_output = np.empty(num_output)
num_sample_per_output.fill(0)
for data, target in test_loader:
data = data.cuda()
data = Variable(data, volatile=True)
output, out_features = model.feature_list(data)
# get hidden features
for i in range(num_output):
out_features[i] = out_features[i].view(out_features[i].size(0),
out_features[i].size(1), -1)
out_features[i] = torch.mean(out_features[i].data, 2)
if num_sample_per_output[i] == 0:
list_features_test[i] = out_features[i]
else:
list_features_test[i] = torch.cat(
(list_features_test[i], out_features[i]), 0)
num_sample_per_output[i] += 1
for out_dist in out_dist_list:
if out_dist == 'FGSM':
test_loader, out_test_loader = data_loader.getFGSM(
args.batch_size, args.dataset, args.net_type)
num_sample_per_output.fill(0)
for data in test_loader:
data = data.cuda()
data = Variable(data, volatile=True)
output, out_features = model.feature_list(data)
# get hidden features
for i in range(num_output):
out_features[i] = out_features[i].view(
out_features[i].size(0), out_features[i].size(1), -1)
out_features[i] = torch.mean(out_features[i].data, 2)
if num_sample_per_output[i] == 0:
list_features_test[i] = out_features[i]
else:
list_features_test[i] = torch.cat(
(list_features_test[i], out_features[i]), 0)
num_sample_per_output[i] += 1
num_sample_per_output = np.empty(num_output)
num_sample_per_output.fill(0)
for data in out_test_loader:
data = data.cuda()
data = Variable(data, requires_grad=True)
output, out_features = model.feature_list(data)
# get hidden features
for i in range(num_output):
out_features[i] = out_features[i].view(
out_features[i].size(0), out_features[i].size(1), -1)
out_features[i] = torch.mean(out_features[i].data, 2)
if num_sample_per_output[i] == 0:
list_features_out[i] = out_features[i]
else:
list_features_out[i] = torch.cat(
(list_features_out[i], out_features[i]), 0)
num_sample_per_output[i] += 1
for i in range(num_output):
sample_class_mean[i] = sample_class_mean[i].cpu()
list_features_test[i] = list_features_test[i].cpu()
list_features_out[i] = list_features_out[i].cpu()
for j in range(args.num_classes):
list_features[i][j] = list_features[i][j].cpu()
else:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
num_sample_per_output.fill(0)
for data, target in out_test_loader:
data, target = data.cuda(), target.cuda()
data, target = Variable(data,
requires_grad=True), Variable(target)
output, out_features = model.feature_list(data)
# get hidden features
for i in range(num_output):
out_features[i] = out_features[i].view(
out_features[i].size(0), out_features[i].size(1), -1)
out_features[i] = torch.mean(out_features[i].data, 2)
if num_sample_per_output[i] == 0:
list_features_out[i] = out_features[i]
else:
list_features_out[i] = torch.cat(
(list_features_out[i], out_features[i]), 0)
num_sample_per_output[i] += 1
for i in range(num_output):
sample_class_mean[i] = sample_class_mean[i].cpu()
list_features_test[i] = list_features_test[i].cpu()
list_features_out[i] = list_features_out[i].cpu()
for j in range(args.num_classes):
list_features[i][j] = list_features[i][j].cpu()
file_name = os.path.join(
'feature_lists', 'feature_lists_{}_{}_{}_Wlinear.pickle'.format(
args.net_type, out_dist, args.dataset))
with open(file_name, 'wb') as f:
pickle.dump([
sample_class_mean, list_features, list_features_test,
list_features_out, A, A_inv, log_abs_det_A_inv
], f)
torch.cuda.manual_seed(args.seed)
kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
print('Load model')
model = models.vgg13()
model.load_state_dict(torch.load(args.pre_trained_net))
print(model)
print('load target data: ', args.dataset)
_, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size,
args.imageSize, args.dataroot)
print('load non target data: ', args.out_dataset)
nt_test_loader = data_loader.getNonTargetDataSet(args.out_dataset,
args.batch_size,
args.imageSize, args.dataroot)
if args.cuda:
model.cuda()
def generate_target():
model.eval()
correct = 0
total = 0
f1 = open('%s/confidence_Base_In.txt' % args.outf, 'w')
for data, target in test_loader:
total += data.size(0)
#vutils.save_image(data, '%s/target_samples.png'%args.outf, normalize=True)
def main():
# set the path to pre-trained model and output
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
out_dist_list = [
'skin_cli', 'skin_derm', 'corrupted', 'corrupted_70', 'imgnet', 'nct'
]
# load networks
if args.net_type == 'densenet_121':
model = densenet_121.Net(models.densenet121(pretrained=False), 8)
ckpt = torch.load("../checkpoints/densenet-121/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
elif args.net_type == 'mobilenet':
model = mobilenet.Net(models.mobilenet_v2(pretrained=False), 8)
ckpt = torch.load("../checkpoints/mobilenet/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
print("Done!")
elif args.net_type == 'resnet_50':
model = resnet_50.Net(models.resnet50(pretrained=False), 8)
ckpt = torch.load("../checkpoints/resnet-50/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
print("Done!")
elif args.net_type == 'vgg_16':
model = vgg_16.Net(models.vgg16_bn(pretrained=False), 8)
ckpt = torch.load("../checkpoints/vgg-16/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
print("Done!")
else:
raise Exception(f"There is no net_type={args.net_type} available.")
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# measure the performance
M_list = [
0, 0.0005, 0.001, 0.0014, 0.002, 0.0024, 0.005, 0.01, 0.05, 0.1, 0.2
]
T_list = [1, 10, 100, 1000]
base_line_list = []
ODIN_best_tnr = [0, 0, 0] * 2
ODIN_best_results = [0, 0, 0] * 2
ODIN_best_temperature = [-1, -1, -1] * 2
ODIN_best_magnitude = [-1, -1, -1] * 2
for T in T_list:
for m in M_list:
magnitude = m
temperature = T
lib_generation.get_posterior(model, args.net_type, test_loader,
magnitude, temperature, args.outf,
True)
out_count = 0
print('Temperature: ' + str(temperature) + ' / noise: ' +
str(magnitude))
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
print('Out-distribution: ' + out_dist)
lib_generation.get_posterior(model, args.net_type,
out_test_loader, magnitude,
temperature, args.outf, False)
if temperature == 1 and magnitude == 0:
test_results = callog.metric(args.outf, ['PoT'])
base_line_list.append(test_results)
else:
val_results = callog.metric(args.outf, ['PoV'])
if ODIN_best_tnr[out_count] < val_results['PoV']['TNR']:
ODIN_best_tnr[out_count] = val_results['PoV']['TNR']
ODIN_best_results[out_count] = callog.metric(
args.outf, ['PoT'])
ODIN_best_temperature[out_count] = temperature
ODIN_best_magnitude[out_count] = magnitude
out_count += 1
# print the results
mtypes = ['TNR', 'AUROC', 'DTACC', 'AUIN', 'AUOUT']
print('Baseline method: in_distribution: ' + args.dataset + '==========')
count_out = 0
for results in base_line_list:
print('out_distribution: ' + out_dist_list[count_out])
for mtype in mtypes:
print(' {mtype:6s}'.format(mtype=mtype), end='')
print('\n{val:6.2f}'.format(val=100. * results['PoT']['TNR']), end='')
print(' {val:6.2f}'.format(val=100. * results['PoT']['AUROC']), end='')
print(' {val:6.2f}'.format(val=100. * results['PoT']['DTACC']), end='')
print(' {val:6.2f}'.format(val=100. * results['PoT']['AUIN']), end='')
print(' {val:6.2f}\n'.format(val=100. * results['PoT']['AUOUT']),
end='')
print('')
count_out += 1
print('ODIN method: in_distribution: ' + args.dataset + '==========')
count_out = 0
for results in ODIN_best_results:
print('out_distribution: ' + out_dist_list[count_out])
for mtype in mtypes:
print(' {mtype:6s}'.format(mtype=mtype), end='')
print('\n{val:6.2f}'.format(val=100. * results['PoT']['TNR']), end='')
print(' {val:6.2f}'.format(val=100. * results['PoT']['AUROC']), end='')
print(' {val:6.2f}'.format(val=100. * results['PoT']['DTACC']), end='')
print(' {val:6.2f}'.format(val=100. * results['PoT']['AUIN']), end='')
print(' {val:6.2f}\n'.format(val=100. * results['PoT']['AUOUT']),
end='')
print('temperature: ' + str(ODIN_best_temperature[count_out]))
print('magnitude: ' + str(ODIN_best_magnitude[count_out]))
print('')
count_out += 1
def main():
# set the path to pre-trained model and output
pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth"
args.outf = args.outf + args.net_type + "_" + args.dataset + "/"
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == "cifar100":
args.num_classes = 100
if args.dataset == "svhn":
out_dist_list = ["cifar10", "imagenet_resize", "lsun_resize"]
else:
out_dist_list = ["svhn", "imagenet_resize", "lsun_resize"]
# load networks
if args.net_type == "densenet":
if args.dataset == "svhn":
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))
)
else:
model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose(
[
transforms.ToTensor(),
transforms.Normalize(
(125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0),
),
]
)
elif args.net_type == "resnet":
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))
)
in_transform = transforms.Compose(
[
transforms.ToTensor(),
transforms.Normalize(
(0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)
),
]
)
model.cuda()
print("load model: " + args.net_type)
# load dataset
print("load target data: ", args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot
)
# measure the performance
M_list = [0, 0.0005, 0.001, 0.0014, 0.002, 0.0024, 0.005, 0.01, 0.05, 0.1, 0.2]
T_list = [1, 10, 100, 1000]
base_line_list = []
ODIN_best_tnr = [0, 0, 0]
ODIN_best_results = [0, 0, 0]
ODIN_best_temperature = [-1, -1, -1]
ODIN_best_magnitude = [-1, -1, -1]
for T in T_list:
for m in M_list:
magnitude = m
temperature = T
lib_generation.get_posterior(
model,
args.net_type,
test_loader,
magnitude,
temperature,
args.outf,
True,
)
out_count = 0
print("Temperature: " + str(temperature) + " / noise: " + str(magnitude))
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot
)
print("Out-distribution: " + out_dist)
lib_generation.get_posterior(
model,
args.net_type,
out_test_loader,
magnitude,
temperature,
args.outf,
False,
)
if temperature == 1 and magnitude == 0:
test_results = callog.metric(args.outf, ["PoT"])
base_line_list.append(test_results)
else:
val_results = callog.metric(args.outf, ["PoV"])
if ODIN_best_tnr[out_count] < val_results["PoV"]["TNR"]:
ODIN_best_tnr[out_count] = val_results["PoV"]["TNR"]
ODIN_best_results[out_count] = callog.metric(args.outf, ["PoT"])
ODIN_best_temperature[out_count] = temperature
ODIN_best_magnitude[out_count] = magnitude
out_count += 1
# print the results
mtypes = ["TNR", "AUROC", "DTACC", "AUIN", "AUOUT"]
print("Baseline method: in_distribution: " + args.dataset + "==========")
count_out = 0
for results in base_line_list:
print("out_distribution: " + out_dist_list[count_out])
for mtype in mtypes:
print(" {mtype:6s}".format(mtype=mtype), end="")
print("\n{val:6.2f}".format(val=100.0 * results["PoT"]["TNR"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUROC"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["DTACC"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUIN"]), end="")
print(" {val:6.2f}\n".format(val=100.0 * results["PoT"]["AUOUT"]), end="")
print("")
count_out += 1
print("ODIN method: in_distribution: " + args.dataset + "==========")
count_out = 0
for results in ODIN_best_results:
print("out_distribution: " + out_dist_list[count_out])
for mtype in mtypes:
print(" {mtype:6s}".format(mtype=mtype), end="")
print("\n{val:6.2f}".format(val=100.0 * results["PoT"]["TNR"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUROC"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["DTACC"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUIN"]), end="")
print(" {val:6.2f}\n".format(val=100.0 * results["PoT"]["AUOUT"]), end="")
print("temperature: " + str(ODIN_best_temperature[count_out]))
print("magnitude: " + str(ODIN_best_magnitude[count_out]))
print("")
count_out += 1
def main():
if args.validation_src == 'FGSM':
if args.dataset == 'svhn':
out_dist_list = [
'cifar10', 'imagenet_resize', 'lsun_resize', 'FGSM'
]
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize', 'FGSM']
else:
if args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
outf_load = os.path.join(args.outf,
args.net_type + '_' + args.dataset + 'RealNVP')
outf = os.path.join(
args.outf, args.net_type + '_' + args.dataset + 'RealNVP_magnitude')
if os.path.isdir(outf) == False:
os.mkdir(outf)
# torch.cuda.manual_seed(0)
torch.cuda.set_device(args.cuda_index)
if args.dataset == 'cifar100':
args.num_classes = 100
else:
args.num_classes = 10
with open(
'feature_lists/feature_lists_{}_imagenet_resize_{}_Wlinear.pickle'.
format(args.net_type, args.dataset), 'rb') as f:
[
sample_class_mean, list_features, list_features_test,
list_features_out, A, A_inv, log_abs_det_A_inv
] = pickle.load(f)
pre_trained_net = args.net_type + '_' + args.dataset + '.pth'
pre_trained_net = os.path.join('pre_trained', pre_trained_net)
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.cuda_index)))
else:
model = torch.load(pre_trained_net, map_location="cpu")
for i, (name, module) in enumerate(model._modules.items()):
module = recursion_change_bn(model)
for m in model.modules():
if 'Conv' in str(type(m)):
setattr(m, 'padding_mode', 'zeros')
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
else:
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.cuda_index)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
print('load model: ' + args.net_type)
model.to(device)
model.eval()
# load dataset
print('load in-data: ', args.dataset)
num_layers = len(sample_class_mean)
for layer in range(num_layers):
num_features = A_inv[layer].shape[0]
half_features = int(num_features / 2)
zeros = np.zeros(half_features)
ones = np.ones(half_features)
right = np.concatenate((zeros, ones), axis=None)
left = np.concatenate((ones, zeros), axis=None)
masks = torch.from_numpy(
np.array([
right, left, right, left, right, left, right, left, right, left
]).astype(np.float32)).cuda()
flow = []
# We reduce the number of neurons in the hidden layers due to GPU memory limitations (11 GB in GTX 2080Ti) - comment out this line for larger GPU memory
length_hidden = reture_length_hidden(layer)
A_layer = torch.tensor(A[layer])
A_inv_layer = torch.tensor(A_inv[layer])
log_abs_det_A_inv_layer = torch.tensor(log_abs_det_A_inv[layer])
for i in range(args.num_classes):
MODEL_FLOW = os.path.join(
outf_load,
'model_{}_layer_{}_residual_flow_{}length_hidden'.format(
args.dataset, layer, length_hidden), 'flow_{}'.format(i))
flow.append(
RealNVP(masks, num_features, length_hidden, A_layer,
A_inv_layer, log_abs_det_A_inv_layer))
flow[i].load_state_dict(torch.load(MODEL_FLOW,
map_location="cuda:{}".format(
args.cuda_index)),
strict=False)
flow[i].to(device)
flow[i].eval()
sample_class_mean_layer = sample_class_mean[layer]
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print('Noise: ' + str(magnitude))
_, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
score_in = lib_generation.get_resflow_score(
test_loader, model, layer, args.num_classes, args.net_type,
sample_class_mean_layer, flow, magnitude)
for out_dist in out_dist_list:
print('load out-data: ', out_dist)
if out_dist == 'FGSM':
test_loader, out_loader = data_loader.getFGSM(
args.batch_size, args.dataset, args.net_type)
score_in = lib_generation.get_resflow_score_FGSM(
test_loader, model, layer, args.num_classes,
args.net_type, sample_class_mean_layer, flow,
magnitude)
score_out = lib_generation.get_resflow_score_FGSM(
out_loader, model, layer, args.num_classes,
args.net_type, sample_class_mean_layer, flow,
magnitude)
else:
out_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
score_out = lib_generation.get_resflow_score(
out_loader, model, layer, args.num_classes,
args.net_type, sample_class_mean_layer, flow,
magnitude)
pram = {
'out_dist': out_dist,
'Network_type': args.net_type,
'Layer': layer,
'Batch_size': args.batch_size,
'cuda_index': args.cuda_index,
'length_hidden': length_hidden,
'dropout': False,
'weight_decay': 0,
'init_zeros': True,
'num_flows': int(len(flow[0].t)),
'magnitude': magnitude,
}
with open(
os.path.join(
outf,
'Residual_flow_%s_%s_layer_%s_%smagnitude.txt' %
(args.dataset, out_dist, layer, magnitude)),
'w') as file:
file.write('date: %s\n' % (datetime.datetime.now()))
file.write(json.dumps(pram))
score_in = np.asarray(score_in, dtype=np.float32)
score_out = np.asarray(score_out, dtype=np.float32)
score_data, score_labels = lib_generation.merge_and_generate_labels(
score_out, score_in)
file_name = os.path.join(
outf, 'Residual_flow_%s_%s_layer_%s_%smagnitude' %
(args.dataset, out_dist, layer, magnitude))
score_data = np.concatenate((score_data, score_labels), axis=1)
np.savez(file_name, score_data, pram)
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
elif args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
elif args.dataset == 'ham10000':
#out_dist_list = ['cifar10', 'imagenet_resize', 'face', 'face_age', 'isic-2017', 'isic-2016']
#out_dist_list = ['cifar10', 'face', 'face_age', 'isic-2017', 'isic-2016']
#out_dist_list = ['cifar10', 'cifar100', 'svhn', 'imagenet_resize', 'lsun_resize', 'face', 'face_age', 'isic-2017', 'isic-2016']
out_dist_list = [
'ham10000-avg-smoothing', 'ham10000-brightness',
'ham10000-contrast', 'ham10000-dilation', 'ham10000-erosion',
'ham10000-med-smoothing', 'ham10000-rotation', 'ham10000-shift'
]
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
elif args.net_type == 'densenet121':
model = DenseNet121(num_classes=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)).state_dict())
in_transform = transforms.Compose([
transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize((0.7630069, 0.5456578, 0.5700767),
(0.14093237, 0.15263236, 0.17000099))
])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print('get sample mean and covariance')
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader)
print('get Mahalanobis scores', num_output)
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print('Noise: ' + str(magnitude))
for i in range(num_output):
print('layer_num', i)
M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \
True, args.net_type, sample_mean, precision, i, magnitude)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
print('Out-distribution: ' + out_dist)
for i in range(num_output):
M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \
False, args.net_type, sample_mean, precision, i, magnitude)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_in)
file_name = os.path.join(
args.outf, 'Mahalanobis_%s_%s_%s.npy' %
(str(magnitude), args.dataset, out_dist))
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
def main():
# set the path to pre-trained model and output
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
out_dist_list = [
'skin_cli', 'skin_derm', 'corrupted', 'corrupted_70', 'imgnet', 'nct',
'final_test'
]
# load networks
if args.net_type == 'densenet_121':
model = densenet_121.Net(models.densenet121(pretrained=False), 8)
ckpt = torch.load("../checkpoints/densenet-121/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
elif args.net_type == 'mobilenet':
model = mobilenet.Net(models.mobilenet_v2(pretrained=False), 8)
ckpt = torch.load("../checkpoints/mobilenet/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
print("Done!")
elif args.net_type == 'resnet_50':
model = resnet_50.Net(models.resnet50(pretrained=False), 8)
ckpt = torch.load("../checkpoints/resnet-50/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
print("Done!")
elif args.net_type == 'vgg_16':
model = vgg_16.Net(models.vgg16_bn(pretrained=False), 8)
ckpt = torch.load("../checkpoints/vgg-16/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
print("Done!")
else:
raise Exception(f"There is no net_type={args.net_type} available.")
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 224, 224).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print('get sample mean and covariance')
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader)
print('get Mahalanobis scores')
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print('Noise: ' + str(magnitude))
for i in range(num_output):
M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \
True, args.net_type, sample_mean, precision, i, magnitude)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
print('Out-distribution: ' + out_dist)
for i in range(num_output):
M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \
False, args.net_type, sample_mean, precision, i, magnitude)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_in)
file_name = os.path.join(
args.outf, 'Mahalanobis_%s_%s_%s.npy' %
(str(magnitude), args.dataset, out_dist))
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
def get_conf(config, device, model, in_transform, train_loader, test_loader):
# get layer list
layer_list = config['exp_params']['feature_layers']
# get feature list
model.eval()
input_dim = config['exp_params']['input_dim'] # 2 x 3 x 32 x 32
if config['exp_params']['dataset'] != 'toy_data':
temp_x = torch.rand(2, input_dim[0], input_dim[1],
input_dim[2]).to(device)
else:
temp_x = torch.rand(2, 2).to(device)
_, temp_list = model.feature_list(temp_x, layer_list)
# ASK : Why this round about way to get feature_list size? Using temp_x etc.
# RESPONSE: number of layers
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
# m_list is the list of noise
m_list = config['exp_params']['noise_params']['m_list']
# calculate confidence components to be sent to regressor
regressor_features = config['exp_params']['regressor_features']
class_mean, class_precision, tied_precision, pca_list, knn_search_list, knn_mean, knn_precision = \
get_inputs_for_computing_regressor_feature(regressor_features, model, config, num_output, feature_list, layer_list, train_loader, device)
print("For in-distribution: {}".format(config['exp_params']['dataset']))
init_reg_in = True
for regressor_feature in regressor_features:
# num_output is the number of layers
for i in range(num_output):
in_dist_input = get_features_for_regressor(
regressor_feature, model, config, test_loader,
config['exp_params']['dataset'], i, True, device, class_mean,
class_precision, tied_precision, pca_list, knn_search_list,
knn_mean, knn_precision)
print("in_dist_input shape: ", in_dist_input.shape)
in_dist_input = np.asarray(in_dist_input, dtype=np.float32)
# ASK: what does score of regression mean?
print("Mean score at layer {} for regression type {}: {}".format(
i, regressor_feature, np.mean(in_dist_input)))
if init_reg_in:
regressor_in_dist_input = in_dist_input.reshape(
(in_dist_input.shape[0], -1))
init_reg_in = False
else:
regressor_in_dist_input = np.concatenate(
(regressor_in_dist_input,
in_dist_input.reshape((in_dist_input.shape[0], -1))),
axis=1)
print("Out-distributions to test agains: ",
config['model_params']['out_dist_list'])
for out_dist in config['model_params']['out_dist_list']:
print('Out-distribution: ' + out_dist)
if out_dist == 'subset_cifar100':
out_test_loader = data_loader.getNonTargetDataSet(
out_dist,
config['trainer_params']['batch_size'],
in_transform,
config['exp_params']['dataroot'],
idx=config['model_params']['out_idx'],
num_oods=config['model_params']['num_ood_samples'])
else:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, config['trainer_params']['batch_size'], in_transform,
config['exp_params']['dataroot'])
init_reg_in = True
for regressor_feature in regressor_features:
# num_output is the number of layers
for i in range(num_output):
out_dist_input = get_features_for_regressor(
regressor_feature, model, config, out_test_loader,
out_dist, i, False, device, class_mean, class_precision,
tied_precision, pca_list, knn_search_list, knn_mean,
knn_precision)
print("out_dist_input shape- ", out_dist_input.shape)
out_dist_input = np.asarray(out_dist_input, dtype=np.float32)
print(
"Mean score at layer {} for regression type {}: {}".format(
i, regressor_feature, np.mean(out_dist_input)))
if init_reg_in:
regressor_out_dist_input = out_dist_input.reshape(
(out_dist_input.shape[0], -1))
init_reg_in = False
else:
regressor_out_dist_input = np.concatenate(
(regressor_out_dist_input,
out_dist_input.reshape(
(out_dist_input.shape[0], -1))),
axis=1)
regressor_in_dist_input = np.asarray(regressor_in_dist_input,
dtype=np.float32)
regressor_out_dist_input = np.asarray(regressor_out_dist_input,
dtype=np.float32)
ood_output, Mahalanobis_labels = lib_generation.merge_and_generate_labels(
regressor_out_dist_input, regressor_in_dist_input)
file_name = os.path.join(
config['logging_params']['outf'], 'Mahalanobis_%s_%s_%s.npy' %
(str(m_list[0]), config['exp_params']['dataset'], out_dist))
ood_output = np.concatenate((ood_output, Mahalanobis_labels), axis=1)
np.save(file_name, ood_output)
return ood_output
def main():
dir_path = os.path.join("experiments", args.dir, "train_classify", "data~"+args.dataset+"+model~"+args.net_type+"+loss~"+str(args.loss))
file_path = os.path.join(dir_path, "results_odd.csv")
with open(file_path, "w") as results_file:
results_file.write(
"EXECUTION,MODEL,IN-DATA,OUT-DATA,LOSS,AD-HOC,SCORE,INFER-LEARN,INFER-TRANS,"
"TNR,AUROC,DTACC,AUIN,AUOUT,CPU_FALSE,CPU_TRUE,GPU_FALSE,GPU_TRUE,TEMPERATURE,MAGNITUDE\n")
args_outf = os.path.join("temporary", args.dir, args.loss, args.net_type + '+' + args.dataset)
if os.path.isdir(args_outf) == False:
os.makedirs(args_outf)
# define number of classes
if args.dataset == 'cifar100':
args.num_classes = 100
elif args.dataset == 'imagenet32':
args.num_classes = 1000
else:
args.num_classes = 10
if args.dataset == 'cifar10':
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
elif args.dataset == 'cifar100':
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
elif args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
if args.dataset == 'cifar10':
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.491, 0.482, 0.446), (0.247, 0.243, 0.261))])
elif args.dataset == 'cifar100':
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.507, 0.486, 0.440), (0.267, 0.256, 0.276))])
elif args.dataset == 'svhn':
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.437, 0.443, 0.472), (0.198, 0.201, 0.197))])
for args.execution in range(1, args.executions + 1):
print("EXECUTION:", args.execution)
pre_trained_net = os.path.join(dir_path, "model" + str(args.execution) + ".pth")
if args.loss.split("_")[0] == "softmax":
loss_first_part = losses.SoftMaxLossFirstPart
scores = ["ES"]
elif args.loss.split("_")[0] == "isomax":
loss_first_part = losses.IsoMaxLossFirstPart
scores = ["ES"]
elif args.loss.split("_")[0] == "isomaxplus":
loss_first_part = losses.IsoMaxPlusLossFirstPart
scores = ["MDS"]
# load networks
if args.net_type == 'densenetbc100':
model = models.DenseNet3(100, int(args.num_classes), loss_first_part=loss_first_part)
elif args.net_type == 'resnet110':
model = models.ResNet110(num_c=args.num_classes, loss_first_part=loss_first_part)
model.load_state_dict(torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target valid data: ', args.dataset)
_, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot)
for score in scores:
print("\n\n\n###############################")
print("###############################")
print("SCORE:", score)
print("###############################")
print("###############################")
base_line_list = []
get_scores(model, test_loader, args_outf, True, score)
out_count = 0
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(out_dist, args.batch_size, in_transform, args.dataroot)
print('Out-distribution: ' + out_dist)
get_scores(model, out_test_loader, args_outf, False, score)
test_results = callog.metric(args_outf, ['PoT'])
base_line_list.append(test_results)
out_count += 1
# print the results
mtypes = ['TNR', 'AUROC', 'DTACC', 'AUIN', 'AUOUT']
print('Baseline method: train in_distribution: ' + args.dataset + '==========')
count_out = 0
for results in base_line_list:
print('out_distribution: '+ out_dist_list[count_out])
for mtype in mtypes:
print(' {mtype:6s}'.format(mtype=mtype), end='')
print('\n{val:6.2f}'.format(val=100.*results['PoT']['TNR']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUROC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['DTACC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUIN']), end='')
print(' {val:6.2f}\n'.format(val=100.*results['PoT']['AUOUT']), end='')
print('')
#Saving odd results:
with open(file_path, "a") as results_file:
results_file.write("{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}\n".format(
str(args.execution), args.net_type, args.dataset, out_dist_list[count_out],
str(args.loss), "NATIVE", score, 'NO', False,
'{:.2f}'.format(100.*results['PoT']['TNR']),
'{:.2f}'.format(100.*results['PoT']['AUROC']),
'{:.2f}'.format(100.*results['PoT']['DTACC']),
'{:.2f}'.format(100.*results['PoT']['AUIN']),
'{:.2f}'.format(100.*results['PoT']['AUOUT']),
0, 0, 0, 0, 1, 0))
count_out += 1
