def main():
"""
Args:
- dataset: name of dataset to attack (expected cifar10/cifar100/svhn)
- attack: type of attack to launch (expected fgsm | deepfool | bim | cwl2)
- batch_size: batch size for data loader
- gpu: gpu index
"""
# load data
num_classes = 100 if args.dataset == 'cifar100' else 10
_, loader = data_loader.getTargetDataSet(args.dataset, args.batch_size,
args.data_path)
# load model
model = models.ResNet34(num_c=num_classes)
model.load_state_dict(
torch.load(NET_PATH, map_location="cuda:" + str(args.gpu)))
model.cuda()
# apply attack
if args.attack == "all":
for attack in ADVERSARIAL:
applyAttack(attack, model, loader, num_classes)
else:
applyAttack(args.attack, model, loader, num_classes)
def main():
"""
Args:
- dataset: name of dataset to attack (expected cifar10/cifar100/svhn)
- attack: type of attack to launch (expected fgsm | deepfool | bim | cwl2)
- batch_size: batch size for data loader
- gpu: gpu index
"""
# load data
if args.dataset == 'malaria':
num_classes = 2
elif args.dataset == 'cifar100':
num_classes = 100
else:
num_classes = 10
_, loader = data_loader.getTargetDataSet(args.dataset, args.batch_size,
args.data_path)
# load model
model = models.get_model(args.model)
net_path = NET_PATH + '/' + args.model + '_' + args.dataset + '.pth'
model.load_state_dict(
torch.load(net_path, map_location="cuda:" + str(args.gpu)))
model.cuda()
# apply attack
if args.attack == "all":
for attack in ADVERSARIAL:
applyAttack(attack, model, loader, num_classes)
else:
applyAttack(args.attack, model, loader, num_classes)
def __init__(self, in_data, net_path, save_path, batch_size=args.batch_size):
"""
Args:
- in_data: name of in-dataset
- net_path: path to pre-trained net weights (corresponding to in_data)
- save_path: output file path, must exist
- batch_size: batch size for data loader (both in/out)
"""
self.in_data = in_data
self.num_classes = 100 if self.in_data == 'cifar100' else 10
self.batch_size = batch_size
self.save_path = save_path
# load training data
self.train_loader, self.in_test_loader = data_loader.getTargetDataSet(
self.in_data, self.batch_size, args.data_path)
# load model
self.model = models.ResNet34(num_c=self.num_classes)
self.model.load_state_dict(torch.load(net_path, map_location = "cuda:" + str(args.gpu)))
self.model.cuda()
# get information about num layers, activation sizes by trying a test input
self.model.eval()
temp_x = Variable(torch.rand(2, 3, 32, 32).cuda())
temp_list = self.model.feature_list(temp_x)[1]
self.num_layers = len(temp_list)
self.activation_sizes = [layer.size(1) for layer in temp_list]
def main():
# make directory to save labels
args.outf = args.outf + args.dataset + '/uniform/' + str(
args.noise_fraction) + '/'
if os.path.isdir(args.outf) == False:
os.makedirs(args.outf)
print('load dataset: ' + args.dataset)
num_classes = 10
if args.dataset == 'cifar100':
num_classes = 100
in_transform = transforms.Compose([transforms.ToTensor(), \
transforms.Normalize((125.3/255, 123.0/255, 113.9/255),\
(63.0/255, 62.1/255.0, 66.7/255.0)),])
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, 200, in_transform, args.dataroot)
# generate index_list to change the labels
train_label = torch.LongTensor(train_loader.dataset.train_labels)
total_index = torch.LongTensor(list(range(train_label.size(0))))
total_selected_index = 0
for index in range(num_classes):
index_list = train_label.eq(index)
num_samples_per_class = sum(index_list)
num_selected_samples = int(num_samples_per_class *
args.noise_fraction / 100)
random_index = torch.randperm(num_samples_per_class)
selected_index = total_index[index_list][
random_index][:num_selected_samples]
if index == 0:
total_selected_index = selected_index
else:
total_selected_index = torch.cat(
(total_selected_index, selected_index), 0)
# change labels
total_new_label = train_loader.dataset.train_labels
for index in total_selected_index:
prev_label = total_new_label[index]
while (True):
new_label = nr.randint(0, num_classes)
if prev_label != new_label:
total_new_label[index] = new_label
break
torch.save(total_new_label, '%s/train_labels.npy' % (args.outf))
def GET_DATA(args):
if args.net_type == 'VggBn':
in_transform = transforms.Compose([transforms.ToTensor(), \
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), \
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ])
elif args.net_type == 'resnet':
in_transform = transforms.Compose([transforms.ToTensor(), \
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ])
if not args.dataset == 'imagenet':
train_loader, val_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
else:
val_info = './data/ImageLog.txt'
train_loader = []
val_loader = Mydataloader([122, 122],
val_info,
args.batch_size,
0,
mode='pixel',
shuffle=True)
return train_loader, val_loader
def main():
if os.path.isdir('feature_lists') == False:
os.mkdir('feature_lists')
if args.dataset == 'cifar100':
args.num_classes = 100
else:
args.num_classes = 10
# load networks
pre_trained_net = args.net_type + '_' + args.dataset + '.pth'
pre_trained_net = os.path.join('pre_trained', pre_trained_net)
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(0)))
else:
model = torch.load(pre_trained_net, map_location="cpu")
for i, (name, module) in enumerate(model._modules.items()):
module = recursion_change_bn(model)
for m in model.modules():
if 'Conv' in str(type(m)):
setattr(m, 'padding_mode', 'zeros')
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(0)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
if args.validation_src == 'FGSM':
if args.dataset == 'svhn':
out_dist_list = [
'cifar10', 'imagenet_resize', 'lsun_resize', 'FGSM'
]
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize', 'FGSM']
else:
if args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
print('load model: ' + args.net_type)
model.to(device)
model.eval()
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extraction
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
correct, total = 0, 0
num_output = len(feature_list)
num_sample_per_class = np.empty(args.num_classes)
num_sample_per_class.fill(0)
list_features = []
list_features_test = []
list_features_out = []
for i in range(num_output):
temp_list = []
list_features_test.append(0)
list_features_out.append(0)
for j in range(args.num_classes):
temp_list.append(0)
list_features.append(temp_list)
for data, target in train_loader:
total += data.size(0)
data = data.cuda()
data = Variable(data, volatile=True)
output, out_features = model.feature_list(data)
# get hidden features
for i in range(num_output):
out_features[i] = out_features[i].view(out_features[i].size(0),
out_features[i].size(1), -1)
out_features[i] = torch.mean(out_features[i].data, 2)
# compute the accuracy
pred = output.data.max(1)[1]
equal_flag = pred.eq(target.cuda()).cpu()
correct += equal_flag.sum()
# construct the sample matrix
for i in range(data.size(0)):
label = target[i]
if num_sample_per_class[label] == 0:
out_count = 0
for out in out_features:
list_features[out_count][label] = out[i].view(1, -1)
out_count += 1
else:
out_count = 0
for out in out_features:
list_features[out_count][label] = torch.cat(
(list_features[out_count][label], out[i].view(1, -1)),
0)
out_count += 1
num_sample_per_class[label] += 1
sample_class_mean = []
out_count = 0
for num_feature in feature_list:
temp_list = torch.Tensor(args.num_classes, int(num_feature)).cuda()
for j in range(args.num_classes):
temp_list[j] = torch.mean(list_features[out_count][j], 0)
sample_class_mean.append(temp_list)
out_count += 1
A = []
A_inv = []
log_abs_det_A_inv = []
for k in range(num_output):
X = 0
for i in range(args.num_classes):
if i == 0:
X = list_features[k][i] - sample_class_mean[k][i]
else:
X = torch.cat(
(X, list_features[k][i] - sample_class_mean[k][i]), 0)
# find inverse
u, s, vh = np.linalg.svd((X.cpu().numpy()) / np.sqrt(X.shape[0]),
full_matrices=False)
covariance_real = np.cov(X.cpu().numpy().T)
valid_indx = s > 1e-5
if (valid_indx.sum() % 2 > 0):
valid_indx[valid_indx.sum() - 1] = False
covriance_cal = np.matmul(
np.matmul(vh[valid_indx, :].transpose(),
np.diag(s[valid_indx]**2)), vh[valid_indx, :])
A_temp = np.matmul(vh[valid_indx, :].transpose(),
np.diag(s[valid_indx]))
A.append(A_temp)
covriance_cal2 = np.matmul(A_temp, A_temp.transpose())
s_inv = 1 / s[valid_indx]
A_inv_temp = np.matmul(np.diag(s_inv), vh[valid_indx, :])
A_inv.append(A_inv_temp)
log_abs_det_A_inv_temp = np.sum(np.log(np.abs(s_inv)))
log_abs_det_A_inv.append(log_abs_det_A_inv_temp)
print('\n Training Accuracy:({:.2f}%)\n'.format(100.0 * int(correct) /
int(total)))
num_sample_per_output = np.empty(num_output)
num_sample_per_output.fill(0)
for data, target in test_loader:
data = data.cuda()
data = Variable(data, volatile=True)
output, out_features = model.feature_list(data)
# get hidden features
for i in range(num_output):
out_features[i] = out_features[i].view(out_features[i].size(0),
out_features[i].size(1), -1)
out_features[i] = torch.mean(out_features[i].data, 2)
if num_sample_per_output[i] == 0:
list_features_test[i] = out_features[i]
else:
list_features_test[i] = torch.cat(
(list_features_test[i], out_features[i]), 0)
num_sample_per_output[i] += 1
for out_dist in out_dist_list:
if out_dist == 'FGSM':
test_loader, out_test_loader = data_loader.getFGSM(
args.batch_size, args.dataset, args.net_type)
num_sample_per_output.fill(0)
for data in test_loader:
data = data.cuda()
data = Variable(data, volatile=True)
output, out_features = model.feature_list(data)
# get hidden features
for i in range(num_output):
out_features[i] = out_features[i].view(
out_features[i].size(0), out_features[i].size(1), -1)
out_features[i] = torch.mean(out_features[i].data, 2)
if num_sample_per_output[i] == 0:
list_features_test[i] = out_features[i]
else:
list_features_test[i] = torch.cat(
(list_features_test[i], out_features[i]), 0)
num_sample_per_output[i] += 1
num_sample_per_output = np.empty(num_output)
num_sample_per_output.fill(0)
for data in out_test_loader:
data = data.cuda()
data = Variable(data, requires_grad=True)
output, out_features = model.feature_list(data)
# get hidden features
for i in range(num_output):
out_features[i] = out_features[i].view(
out_features[i].size(0), out_features[i].size(1), -1)
out_features[i] = torch.mean(out_features[i].data, 2)
if num_sample_per_output[i] == 0:
list_features_out[i] = out_features[i]
else:
list_features_out[i] = torch.cat(
(list_features_out[i], out_features[i]), 0)
num_sample_per_output[i] += 1
for i in range(num_output):
sample_class_mean[i] = sample_class_mean[i].cpu()
list_features_test[i] = list_features_test[i].cpu()
list_features_out[i] = list_features_out[i].cpu()
for j in range(args.num_classes):
list_features[i][j] = list_features[i][j].cpu()
else:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
num_sample_per_output.fill(0)
for data, target in out_test_loader:
data, target = data.cuda(), target.cuda()
data, target = Variable(data,
requires_grad=True), Variable(target)
output, out_features = model.feature_list(data)
# get hidden features
for i in range(num_output):
out_features[i] = out_features[i].view(
out_features[i].size(0), out_features[i].size(1), -1)
out_features[i] = torch.mean(out_features[i].data, 2)
if num_sample_per_output[i] == 0:
list_features_out[i] = out_features[i]
else:
list_features_out[i] = torch.cat(
(list_features_out[i], out_features[i]), 0)
num_sample_per_output[i] += 1
for i in range(num_output):
sample_class_mean[i] = sample_class_mean[i].cpu()
list_features_test[i] = list_features_test[i].cpu()
list_features_out[i] = list_features_out[i].cpu()
for j in range(args.num_classes):
list_features[i][j] = list_features[i][j].cpu()
file_name = os.path.join(
'feature_lists', 'feature_lists_{}_{}_{}_Wlinear.pickle'.format(
args.net_type, out_dist, args.dataset))
with open(file_name, 'wb') as f:
pickle.dump([
sample_class_mean, list_features, list_features_test,
list_features_out, A, A_inv, log_abs_det_A_inv
], f)
def main():
dir_path = os.path.join("experiments", args.dir, "train_classify", "data~"+args.dataset+"+model~"+args.net_type+"+loss~"+str(args.loss))
file_path = os.path.join(dir_path, "results_odd.csv")
with open(file_path, "w") as results_file:
results_file.write(
"EXECUTION,MODEL,IN-DATA,OUT-DATA,LOSS,AD-HOC,SCORE,INFER-LEARN,INFER-TRANS,"
"TNR,AUROC,DTACC,AUIN,AUOUT,CPU_FALSE,CPU_TRUE,GPU_FALSE,GPU_TRUE,TEMPERATURE,MAGNITUDE\n")
args_outf = os.path.join("temporary", args.dir, args.loss, args.net_type + '+' + args.dataset)
if os.path.isdir(args_outf) == False:
os.makedirs(args_outf)
# define number of classes
if args.dataset == 'cifar100':
args.num_classes = 100
elif args.dataset == 'imagenet32':
args.num_classes = 1000
else:
args.num_classes = 10
if args.dataset == 'cifar10':
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
elif args.dataset == 'cifar100':
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
elif args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
if args.dataset == 'cifar10':
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.491, 0.482, 0.446), (0.247, 0.243, 0.261))])
elif args.dataset == 'cifar100':
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.507, 0.486, 0.440), (0.267, 0.256, 0.276))])
elif args.dataset == 'svhn':
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.437, 0.443, 0.472), (0.198, 0.201, 0.197))])
for args.execution in range(1, args.executions + 1):
print("EXECUTION:", args.execution)
pre_trained_net = os.path.join(dir_path, "model" + str(args.execution) + ".pth")
if args.loss.split("_")[0] == "softmax":
loss_first_part = losses.SoftMaxLossFirstPart
scores = ["ES"]
elif args.loss.split("_")[0] == "isomax":
loss_first_part = losses.IsoMaxLossFirstPart
scores = ["ES"]
elif args.loss.split("_")[0] == "isomaxplus":
loss_first_part = losses.IsoMaxPlusLossFirstPart
scores = ["MDS"]
# load networks
if args.net_type == 'densenetbc100':
model = models.DenseNet3(100, int(args.num_classes), loss_first_part=loss_first_part)
elif args.net_type == 'resnet110':
model = models.ResNet110(num_c=args.num_classes, loss_first_part=loss_first_part)
model.load_state_dict(torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target valid data: ', args.dataset)
_, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot)
for score in scores:
print("\n\n\n###############################")
print("###############################")
print("SCORE:", score)
print("###############################")
print("###############################")
base_line_list = []
get_scores(model, test_loader, args_outf, True, score)
out_count = 0
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(out_dist, args.batch_size, in_transform, args.dataroot)
print('Out-distribution: ' + out_dist)
get_scores(model, out_test_loader, args_outf, False, score)
test_results = callog.metric(args_outf, ['PoT'])
base_line_list.append(test_results)
out_count += 1
# print the results
mtypes = ['TNR', 'AUROC', 'DTACC', 'AUIN', 'AUOUT']
print('Baseline method: train in_distribution: ' + args.dataset + '==========')
count_out = 0
for results in base_line_list:
print('out_distribution: '+ out_dist_list[count_out])
for mtype in mtypes:
print(' {mtype:6s}'.format(mtype=mtype), end='')
print('\n{val:6.2f}'.format(val=100.*results['PoT']['TNR']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUROC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['DTACC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUIN']), end='')
print(' {val:6.2f}\n'.format(val=100.*results['PoT']['AUOUT']), end='')
print('')
#Saving odd results:
with open(file_path, "a") as results_file:
results_file.write("{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}\n".format(
str(args.execution), args.net_type, args.dataset, out_dist_list[count_out],
str(args.loss), "NATIVE", score, 'NO', False,
'{:.2f}'.format(100.*results['PoT']['TNR']),
'{:.2f}'.format(100.*results['PoT']['AUROC']),
'{:.2f}'.format(100.*results['PoT']['DTACC']),
'{:.2f}'.format(100.*results['PoT']['AUIN']),
'{:.2f}'.format(100.*results['PoT']['AUOUT']),
0, 0, 0, 0, 1, 0))
count_out += 1
def main():
run_dir = args.run_dir
run_name = run_dir.name
matches = runs_regex.match(run_name)
dataset = matches.group(1)
net_type = matches.group(2)
ae_type = matches.group(3)
args.num_classes = 100 if dataset == 'cifar100' else 10
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + net_type + '_' + dataset + '.pth'
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
device = f'cuda:{args.gpu}'
# load networks
if net_type == 'densenet':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print('load model: ' + net_type)
# load dataset
print('load target data: ', dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
dataset, args.batch_size, in_transform, args.dataroot)
# get the feature maps' sizes
model.eval()
temp_x = torch.rand(1, 3, 32, 32).cuda()
feature_maps = model.feature_list(temp_x)[1]
feature_map_sizes = []
for out in feature_maps:
feature_map_sizes.append(out.size()[1:])
criterion = torch.nn.CrossEntropyLoss()
ae_criterion = torch.nn.MSELoss(reduction='none')
ae_type = ae_type
filters = [128, 128, 128]
latent_size = 64
bn = False
if ae_type == 'vae':
lamb = 1e-7
elif ae_type == 'wae':
lamb = 1e-4
elif ae_type == 'waegan':
if net_type == 'densenet':
lamb = [5e-5, 1e-5, 1e-5, 1e-3]
elif net_type == 'resnet':
lamb = [5e-5, 5e-6, 5e-6, 2e-5, 5e-4]
else:
lamb = None
if ae_type == 'waegan':
ae_models = [
build_ae_for_size(ae_type, sizes, filters, latent_size, lam, bn)
for sizes, lam in zip(feature_map_sizes, lamb)
]
else:
ae_models = [
build_ae_for_size(ae_type, sizes, filters, latent_size, lamb, bn)
for sizes in feature_map_sizes
]
# read in model and autoencoder models
models_filename = f'{run_name}/models.pth'
state_dict = torch.load(models_filename)
for i, _ in enumerate(ae_models):
if isinstance(ae_models[i], torch.nn.Module):
ae_models[i].load_state_dict(state_dict[f'model_{i}'])
else:
for j in range(len(ae_models[i])):
ae_models[i][j].load_state_dict(state_dict[f'model_{i}'][j])
for ae_model in ae_models:
if isinstance(ae_model, torch.nn.Module):
ae_model.eval()
else:
for m in ae_model:
m.eval()
# read in final unsupervised model
classifier_model = 'IF'
run_dir = args.run_dir
name_prefix = 'latent_' if args.latent else ''
name_prefix += 'rec_error_' if args.rec_error else ''
name_prefix += 'both_' if args.both else ''
model_filename = str(run_dir /
f'{name_prefix}final_cv_{classifier_model}.joblib')
assert Path(model_filename).exists()
if_classfier = joblib.load(model_filename)
pgd_iters = [(i + 1) * 10 for i in range(20)]
results_file = run_dir / 'increasing_strength_pgd.pickle'
if results_file.exists():
with open(results_file, 'rb') as results_fd:
results = pickle.load(results_fd)
else:
results = {}
for iters in pgd_iters:
# calculate only if result is not present already
key = f'PGD{iters}'
if key not in results:
# read in adversarial AE encoded samples
dataset_names = [f'clean_{key}', f'adv_{key}', f'noisy_{key}']
datasets = {}
for name in dataset_names:
dataset_path = run_dir / f'ae_encoded_{name}.npy'
if dataset_path.exists():
datasets[name] = np.load(str(dataset_path))
else:
print(f'{dataset_path} is missing!')
test_size = len(datasets[f'clean_{key}'])
X_test = np.concatenate([
datasets[f'clean_{key}'],
datasets[f'adv_{key}'],
datasets[f'noisy_{key}'],
])
y_test = np.concatenate([
np.ones(test_size),
np.zeros(test_size),
np.ones(test_size),
])
# classify with the unsupervised classifier
y_pred = if_classfier.predict(X_test)
try:
y_scores = if_classfier.decision_function(X_test)
except:
y_scores = if_classfier.predict_proba(X_test)[0]
# calculate scores
acc = accuracy_score(y_test, y_pred)
auroc = roc_auc_score(y_test, y_scores)
results[f'acc_{key}'] = acc
results[f'auroc_{key}'] = auroc
# save results
with open(results_file, 'wb') as results_fd:
pickle.dump(results, results_fd)
# gather data to numpy arrays
xs = np.array(pgd_iters)
ys = np.zeros(len(xs))
ys_acc = np.zeros(len(xs))
for i, iters in enumerate(pgd_iters):
key = f'PGD{iters}'
ys[i] = results[f'auroc_{key}']
ys_acc[i] = results[f'acc_{key}']
# generate plots
plot_filename = run_dir / 'increasing_strength_pgd.png'
fig, ax1 = plt.subplots(1, 1, figsize=(16, 9))
color = 'tab:red'
ax1.set_xlabel('PGD iterations')
ax1.set_ylabel('AUROC', color=color)
ax1.plot(xs, ys, color=color)
ax1.tick_params(axis='y', labelcolor=color)
ax2 = ax1.twinx() # instantiate a second axes that shares the same x-axis
color = 'tab:blue'
ax2.set_ylabel('Accuracy',
color=color) # we already handled the x-label with ax1
ax2.plot(xs, ys_acc, color=color)
ax2.tick_params(axis='y', labelcolor=color)
fig.tight_layout() # otherwise the right y-label is slightly clipped
print(f'Writing {plot_filename}')
fig.savefig(plot_filename, bbox_inches='tight')
plot_filename = run_dir / 'auroc_increasing_strength_pgd.png'
fig, ax1 = plt.subplots(1, 1, figsize=(16, 9))
ax1.set_xlabel('PGD iterations')
ax1.set_ylabel('AUROC')
x_ticks = xs[::2]
ax1.set_xticks(x_ticks)
ax1.set_xticklabels([str(x) for x in x_ticks])
ax1.plot(xs, ys, marker='o', linestyle='solid', markeredgecolor='b')
fig.tight_layout() # otherwise the right y-label is slightly clipped
print(f'Writing {plot_filename}')
fig.savefig(plot_filename, bbox_inches='tight')
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
if args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
################################ edits
print("Calculate SVD before getting sample mean and variance")
# svd_result= lib_generation.get_pca(model, args.num_classes, feature_list, train_loader)
# lib_generation.get_pca_incremental(model, args.num_classes, feature_list, train_loader,args)
svd_result = None
print('get sample mean and covariance')
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader, svd_result, args)
################################ edits_end_sample_generator
print('get Mahalanobis scores')
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print('Noise: ' + str(magnitude))
for i in range(num_output):
M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \
True, args.net_type, sample_mean, precision, i, magnitude,svd_result,args)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
print('Out-distribution: ' + out_dist)
for i in range(num_output):
M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \
False, args.net_type, sample_mean, precision, i, magnitude,svd_result,args)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_in)
file_name = os.path.join(
args.outf, 'Mahalanobis_%s_%s_%s.npy' %
(str(magnitude), args.dataset, out_dist))
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
import time
dataset = 'svhn'
dataroot = './data'
batch_size = 128
imageSize = 32
data_cut_len = 10
print('Load GAN')
nz = 100
netG = Generator(1, nz, 64, 3, train=False)
netG.load_state_dict(torch.load('./netG_epoch_100.pth'))
netG.cuda()
print('load data')
train_loader, test_loader = data_loader.getTargetDataSet(
dataset, batch_size, imageSize, dataroot)
netG.eval()
with torch.no_grad():
for batch_idx, (data, target) in enumerate(train_loader):
if (batch_idx == data_cut_len):
break
if batch_idx == 0:
data_list = [data]
target_list = [target]
else:
data_list.append(data)
target_list.append(target)
svhn_data_all = torch.cat(data_list, dim=0).cpu()
svhn_target_all = [0 for _ in range(svhn_data_all.size(0))]
def main():
# set the path to pre-trained model and output
pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth"
args.outf = args.outf + args.net_type + "_" + args.dataset + "/"
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == "cifar100":
args.num_classes = 100
if args.dataset == "svhn":
out_dist_list = ["cifar10", "imagenet_resize", "lsun_resize"]
else:
out_dist_list = ["svhn", "imagenet_resize", "lsun_resize"]
# load networks
if args.net_type == "densenet":
if args.dataset == "svhn":
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
(125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0),
),
])
elif args.net_type == "resnet":
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print("load model: " + args.net_type)
# load dataset
print("load target data: ", args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print("get sample mean and covariance")
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader)
print("get Mahalanobis scores")
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print("Noise: " + str(magnitude))
for i in range(num_output):
M_in = lib_generation.get_Mahalanobis_score(
model,
test_loader,
args.num_classes,
args.outf,
True,
args.net_type,
sample_mean,
precision,
i,
magnitude,
)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
print("Out-distribution: " + out_dist)
for i in range(num_output):
M_out = lib_generation.get_Mahalanobis_score(
model,
out_test_loader,
args.num_classes,
args.outf,
False,
args.net_type,
sample_mean,
precision,
i,
magnitude,
)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
(
Mahalanobis_data,
Mahalanobis_labels,
) = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_in)
file_name = os.path.join(
args.outf,
"Mahalanobis_%s_%s_%s.npy" %
(str(magnitude), args.dataset, out_dist),
)
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
def main():
if args.validation_src == 'FGSM':
if args.dataset == 'svhn':
out_dist_list = [
'cifar10', 'imagenet_resize', 'lsun_resize', 'FGSM'
]
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize', 'FGSM']
else:
if args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
outf_load = os.path.join(args.outf,
args.net_type + '_' + args.dataset + 'RealNVP')
outf = os.path.join(
args.outf, args.net_type + '_' + args.dataset + 'RealNVP_magnitude')
if os.path.isdir(outf) == False:
os.mkdir(outf)
# torch.cuda.manual_seed(0)
torch.cuda.set_device(args.cuda_index)
if args.dataset == 'cifar100':
args.num_classes = 100
else:
args.num_classes = 10
with open(
'feature_lists/feature_lists_{}_imagenet_resize_{}_Wlinear.pickle'.
format(args.net_type, args.dataset), 'rb') as f:
[
sample_class_mean, list_features, list_features_test,
list_features_out, A, A_inv, log_abs_det_A_inv
] = pickle.load(f)
pre_trained_net = args.net_type + '_' + args.dataset + '.pth'
pre_trained_net = os.path.join('pre_trained', pre_trained_net)
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.cuda_index)))
else:
model = torch.load(pre_trained_net, map_location="cpu")
for i, (name, module) in enumerate(model._modules.items()):
module = recursion_change_bn(model)
for m in model.modules():
if 'Conv' in str(type(m)):
setattr(m, 'padding_mode', 'zeros')
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
else:
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.cuda_index)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
print('load model: ' + args.net_type)
model.to(device)
model.eval()
# load dataset
print('load in-data: ', args.dataset)
num_layers = len(sample_class_mean)
for layer in range(num_layers):
num_features = A_inv[layer].shape[0]
half_features = int(num_features / 2)
zeros = np.zeros(half_features)
ones = np.ones(half_features)
right = np.concatenate((zeros, ones), axis=None)
left = np.concatenate((ones, zeros), axis=None)
masks = torch.from_numpy(
np.array([
right, left, right, left, right, left, right, left, right, left
]).astype(np.float32)).cuda()
flow = []
# We reduce the number of neurons in the hidden layers due to GPU memory limitations (11 GB in GTX 2080Ti) - comment out this line for larger GPU memory
length_hidden = reture_length_hidden(layer)
A_layer = torch.tensor(A[layer])
A_inv_layer = torch.tensor(A_inv[layer])
log_abs_det_A_inv_layer = torch.tensor(log_abs_det_A_inv[layer])
for i in range(args.num_classes):
MODEL_FLOW = os.path.join(
outf_load,
'model_{}_layer_{}_residual_flow_{}length_hidden'.format(
args.dataset, layer, length_hidden), 'flow_{}'.format(i))
flow.append(
RealNVP(masks, num_features, length_hidden, A_layer,
A_inv_layer, log_abs_det_A_inv_layer))
flow[i].load_state_dict(torch.load(MODEL_FLOW,
map_location="cuda:{}".format(
args.cuda_index)),
strict=False)
flow[i].to(device)
flow[i].eval()
sample_class_mean_layer = sample_class_mean[layer]
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print('Noise: ' + str(magnitude))
_, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
score_in = lib_generation.get_resflow_score(
test_loader, model, layer, args.num_classes, args.net_type,
sample_class_mean_layer, flow, magnitude)
for out_dist in out_dist_list:
print('load out-data: ', out_dist)
if out_dist == 'FGSM':
test_loader, out_loader = data_loader.getFGSM(
args.batch_size, args.dataset, args.net_type)
score_in = lib_generation.get_resflow_score_FGSM(
test_loader, model, layer, args.num_classes,
args.net_type, sample_class_mean_layer, flow,
magnitude)
score_out = lib_generation.get_resflow_score_FGSM(
out_loader, model, layer, args.num_classes,
args.net_type, sample_class_mean_layer, flow,
magnitude)
else:
out_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
score_out = lib_generation.get_resflow_score(
out_loader, model, layer, args.num_classes,
args.net_type, sample_class_mean_layer, flow,
magnitude)
pram = {
'out_dist': out_dist,
'Network_type': args.net_type,
'Layer': layer,
'Batch_size': args.batch_size,
'cuda_index': args.cuda_index,
'length_hidden': length_hidden,
'dropout': False,
'weight_decay': 0,
'init_zeros': True,
'num_flows': int(len(flow[0].t)),
'magnitude': magnitude,
}
with open(
os.path.join(
outf,
'Residual_flow_%s_%s_layer_%s_%smagnitude.txt' %
(args.dataset, out_dist, layer, magnitude)),
'w') as file:
file.write('date: %s\n' % (datetime.datetime.now()))
file.write(json.dumps(pram))
score_in = np.asarray(score_in, dtype=np.float32)
score_out = np.asarray(score_out, dtype=np.float32)
score_data, score_labels = lib_generation.merge_and_generate_labels(
score_out, score_in)
file_name = os.path.join(
outf, 'Residual_flow_%s_%s_layer_%s_%smagnitude' %
(args.dataset, out_dist, layer, magnitude))
score_data = np.concatenate((score_data, score_labels), axis=1)
np.savez(file_name, score_data, pram)
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
elif args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
elif args.dataset == 'ham10000':
#out_dist_list = ['cifar10', 'imagenet_resize', 'face', 'face_age', 'isic-2017', 'isic-2016']
#out_dist_list = ['cifar10', 'face', 'face_age', 'isic-2017', 'isic-2016']
#out_dist_list = ['cifar10', 'cifar100', 'svhn', 'imagenet_resize', 'lsun_resize', 'face', 'face_age', 'isic-2017', 'isic-2016']
out_dist_list = [
'ham10000-avg-smoothing', 'ham10000-brightness',
'ham10000-contrast', 'ham10000-dilation', 'ham10000-erosion',
'ham10000-med-smoothing', 'ham10000-rotation', 'ham10000-shift'
]
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
elif args.net_type == 'densenet121':
model = DenseNet121(num_classes=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)).state_dict())
in_transform = transforms.Compose([
transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize((0.7630069, 0.5456578, 0.5700767),
(0.14093237, 0.15263236, 0.17000099))
])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print('get sample mean and covariance')
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader)
print('get Mahalanobis scores', num_output)
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print('Noise: ' + str(magnitude))
for i in range(num_output):
print('layer_num', i)
M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \
True, args.net_type, sample_mean, precision, i, magnitude)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
print('Out-distribution: ' + out_dist)
for i in range(num_output):
M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \
False, args.net_type, sample_mean, precision, i, magnitude)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_in)
file_name = os.path.join(
args.outf, 'Mahalanobis_%s_%s_%s.npy' %
(str(magnitude), args.dataset, out_dist))
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
def main():
# set the path to pre-trained model and output
pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth"
args.outf = args.outf + args.net_type + "_" + args.dataset + "/"
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == "cifar100":
args.num_classes = 100
# load networks
if args.net_type == "densenet":
if args.dataset == "svhn":
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
(125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0),
),
])
elif args.net_type == "resnet":
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print("load model: " + args.net_type)
# load dataset
print("load target data: ", args.dataset)
train_loader, _ = data_loader.getTargetDataSet(args.dataset,
args.batch_size,
in_transform, args.dataroot)
test_clean_data = torch.load(args.outf + "clean_data_%s_%s_%s.pth" %
(args.net_type, args.dataset, args.adv_type))
test_adv_data = torch.load(args.outf + "adv_data_%s_%s_%s.pth" %
(args.net_type, args.dataset, args.adv_type))
test_noisy_data = torch.load(args.outf + "noisy_data_%s_%s_%s.pth" %
(args.net_type, args.dataset, args.adv_type))
test_label = torch.load(args.outf + "label_%s_%s_%s.pth" %
(args.net_type, args.dataset, args.adv_type))
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print("get sample mean and covariance")
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader)
print("get LID scores")
LID, LID_adv, LID_noisy = lib_generation.get_LID(model, test_clean_data,
test_adv_data,
test_noisy_data,
test_label, num_output)
overlap_list = [10, 20, 30, 40, 50, 60, 70, 80, 90]
list_counter = 0
for overlap in overlap_list:
Save_LID = np.asarray(LID[list_counter], dtype=np.float32)
Save_LID_adv = np.asarray(LID_adv[list_counter], dtype=np.float32)
Save_LID_noisy = np.asarray(LID_noisy[list_counter], dtype=np.float32)
Save_LID_pos = np.concatenate((Save_LID, Save_LID_noisy))
LID_data, LID_labels = lib_generation.merge_and_generate_labels(
Save_LID_adv, Save_LID_pos)
file_name = os.path.join(
args.outf,
"LID_%s_%s_%s.npy" % (overlap, args.dataset, args.adv_type))
LID_data = np.concatenate((LID_data, LID_labels), axis=1)
np.save(file_name, LID_data)
list_counter += 1
print("get Mahalanobis scores")
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print("\nNoise: " + str(magnitude))
for i in range(num_output):
M_in = lib_generation.get_Mahalanobis_score_adv(
model,
test_clean_data,
test_label,
args.num_classes,
args.outf,
args.net_type,
sample_mean,
precision,
i,
magnitude,
)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for i in range(num_output):
M_out = lib_generation.get_Mahalanobis_score_adv(
model,
test_adv_data,
test_label,
args.num_classes,
args.outf,
args.net_type,
sample_mean,
precision,
i,
magnitude,
)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
for i in range(num_output):
M_noisy = lib_generation.get_Mahalanobis_score_adv(
model,
test_noisy_data,
test_label,
args.num_classes,
args.outf,
args.net_type,
sample_mean,
precision,
i,
magnitude,
)
M_noisy = np.asarray(M_noisy, dtype=np.float32)
if i == 0:
Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1))
else:
Mahalanobis_noisy = np.concatenate(
(Mahalanobis_noisy, M_noisy.reshape(
(M_noisy.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32)
Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy))
Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_pos)
file_name = os.path.join(
args.outf,
"Mahalanobis_%s_%s_%s.npy" %
(str(magnitude), args.dataset, args.adv_type),
)
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
# load networks
if args.net_type == 'densenet':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
test_clean_data, test_adv_data, test_noisy_data, test_label = {}, {}, {}, {}
clean_loaders, adv_loaders, noisy_loaders = {}, {}, {}
adv_types = ['FGSM', 'BIM', 'DeepFool', 'CWL2', 'PGD100'
] if args.adv_type is None else [args.adv_type]
for adv_type in adv_types:
test_clean_data[adv_type] = torch.load(
args.outf + 'clean_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
test_adv_data[adv_type] = torch.load(
args.outf + 'adv_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
test_noisy_data[adv_type] = torch.load(
args.outf + 'noisy_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
test_label[adv_type] = torch.load(
args.outf + 'label_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
clean_loaders[adv_type] = torch.utils.data.DataLoader(
test_clean_data[adv_type], batch_size=args.batch_size)
adv_loaders[adv_type] = torch.utils.data.DataLoader(
test_adv_data[adv_type], batch_size=args.batch_size)
noisy_loaders[adv_type] = torch.utils.data.DataLoader(
test_noisy_data[adv_type], batch_size=args.batch_size)
# get the feature maps' sizes
model.eval()
temp_x = torch.rand(1, 3, 32, 32).cuda()
feature_maps = model.feature_list(temp_x)[1]
feature_map_sizes = []
for out in feature_maps:
feature_map_sizes.append(out.size()[1:])
opt_class = torch.optim.Adam
criterion = torch.nn.MSELoss(reduction='none')
ae_type = args.ae_type
filters = [128, 128, 128]
arch_text = str(filters).replace(', ', '_').replace('(',
'').replace(')', '')
latent_size = 64
bn = False
if ae_type == 'vae':
lamb = 1e-7
elif ae_type == 'wae':
lamb = 1e-4
elif ae_type == 'waegan':
if args.net_type == 'densenet':
lamb = [5e-5, 1e-5, 1e-5, 1e-3]
elif args.net_type == 'resnet':
lamb = [5e-5, 5e-6, 5e-6, 2e-5, 5e-4]
else:
lamb = None
lr = 1e-3
for run_n in range(args.runs):
if ae_type == 'waegan':
ae_models = [
build_ae_for_size(ae_type,
sizes,
filters,
latent_size,
lam,
bn,
info=True)
for sizes, lam in zip(feature_map_sizes, lamb)
]
ae_opts = []
for ae_model in ae_models:
ae_opts.append(
tuple([
opt_class(ae_model[i].parameters(), lr)
for i in range(len(ae_model))
]))
else:
ae_models = [
build_ae_for_size(ae_type,
sizes,
filters,
latent_size,
lamb,
bn,
info=True) for sizes in feature_map_sizes
]
ae_opts = [
opt_class(ae_model.parameters(), lr) for ae_model in ae_models
]
run_name = (f'{args.dataset}_{args.net_type}_deep_{ae_type}_arch_{arch_text}_bn_{bn}' \
f'_latent_{latent_size}_lamb_{lamb}_lr_{lr}_bs_{args.batch_size}_epochs_{args.epochs}_{run_n}') \
.replace('.', '_').replace(',', '_')
print(f'Run name: {run_name}')
writer = SummaryWriter(run_name)
for ae_model in ae_models:
if isinstance(ae_model, torch.nn.Module):
ae_model.train()
else:
for m in ae_model:
m.train()
models_filename = f'{run_name}/models.pth'
models_file = Path(models_filename)
if not models_file.exists():
current_x = 0
with tqdm(initial=current_x, unit_scale=True,
dynamic_ncols=True) as pbar:
for epoch in range(args.epochs):
for ae_model, ae_opt in zip(ae_models, ae_opts):
if isinstance(ae_model, tuple):
encoder_o, decoder_o, discriminator_o = ae_opt
custom_schedule(encoder_o, epoch)
custom_schedule(decoder_o, epoch)
custom_schedule(discriminator_o, epoch)
elif isinstance(ae_model, ConvWAE):
custom_schedule(ae_opt, epoch)
elif isinstance(ae_model, ConvVAE):
custom_schedule(ae_opt, epoch)
for X, _ in train_loader:
X = X.cuda()
y_pred, feature_list = model.feature_list(X)
for i, (feature_map, ae_model, ae_opt) in enumerate(
zip(feature_list, ae_models, ae_opts)):
feature_map = feature_map.detach()
if isinstance(ae_model, tuple):
encoder, decoder, discriminator = ae_model
encoder_o, decoder_o, discriminator_o = ae_opt
z = encoder(feature_map)
fm_recon = decoder(z)
# train discriminator
ones = torch.ones((feature_map.size(0), 1),
device='cuda')
zeros = torch.zeros((feature_map.size(0), 1),
device='cuda')
z_sigma = encoder.z_var**0.5
z_fake = torch.empty(
(X.size(0), discriminator.z_dim),
device='cuda').normal_(std=z_sigma)
dis_fake = discriminator(z_fake)
dis_real = discriminator(z)
fake_loss = F.binary_cross_entropy_with_logits(
dis_fake, ones)
real_loss = F.binary_cross_entropy_with_logits(
dis_real, zeros)
d_loss = discriminator.lamb * (fake_loss +
real_loss)
discriminator_o.zero_grad()
d_loss.backward(retain_graph=True)
discriminator_o.step()
# train autoencoder
# rec_losses = criterion(fm_recon, feature_map).sum(dim=(1, 2, 3))
rec_losses = criterion(
fm_recon, feature_map).mean(dim=(1, 2, 3))
rec_loss = rec_losses.mean()
ae_d_loss = F.binary_cross_entropy_with_logits(
dis_real, ones)
ae_loss = rec_loss + discriminator.lamb * ae_d_loss
encoder_o.zero_grad()
decoder_o.zero_grad()
ae_loss.backward()
encoder_o.step()
decoder_o.step()
writer.add_scalar(f'AE {i}/AE Loss',
ae_loss.item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/AE Reconstruction loss',
rec_loss.item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/AE discriminator loss',
ae_d_loss.item(),
global_step=current_x)
writer.add_scalar(f'AE {i}/Discriminator loss',
d_loss.item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/Discriminator fake loss',
fake_loss.item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/Discriminator mean real output',
torch.sigmoid(dis_real).mean().item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/Discriminator mean fake output',
torch.sigmoid(dis_fake).mean().item(),
global_step=current_x)
else:
if isinstance(ae_model, ConvVAE):
fm_recon, mu, logvar, z = ae_model.forward(
feature_map)
loss, rec_loss, div_loss = ae_model.calculate_loss(
feature_map, fm_recon, criterion, mu,
logvar)
elif isinstance(ae_model, ConvWAE):
fm_recon, z = ae_model.forward(feature_map)
z_sigma = ae_model.z_var**0.5
z_gen = torch.empty(
(feature_map.size(0),
ae_model.z_dim)).normal_(
std=z_sigma).cuda()
loss, rec_loss, dis_loss = ae_model.calculate_loss(
feature_map, fm_recon, criterion,
z_gen, z)
else:
fm_recon, z = ae_model.forward(feature_map)
loss, rec_loss = ae_model.calculate_loss(
feature_map, fm_recon, criterion)
ae_opt.zero_grad()
loss.backward()
ae_opt.step()
if isinstance(ae_model, ConvVAE):
writer.add_scalar(f'AE {i}/Loss',
loss.item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/Reconstruction loss',
rec_loss.mean().item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/Divergence loss',
div_loss.item(),
global_step=current_x)
elif isinstance(ae_model, ConvWAE):
writer.add_scalar(f'AE {i}/Loss',
loss.item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/Reconstruction loss',
rec_loss.mean().item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/Discrepancy loss',
dis_loss.item(),
global_step=current_x)
else:
writer.add_scalar(f'AE {i}/Loss',
loss.item(),
global_step=current_x)
current_x += 1
pbar.set_description(f'Epoch {epoch}')
pbar.update()
# save models
state_dict = {}
for i, ae_model in enumerate(ae_models):
if isinstance(ae_model, torch.nn.Module):
state_dict[f'model_{i}'] = ae_model.state_dict()
else:
state_dict[f'model_{i}'] = tuple([
ae_model[i].state_dict() for i in range(len(ae_model))
])
torch.save(state_dict, models_filename)
else:
state_dict = torch.load(models_filename)
for i, _ in enumerate(ae_models):
if isinstance(ae_models[i], torch.nn.Module):
ae_models[i].load_state_dict(state_dict[f'model_{i}'])
else:
for j in range(len(ae_models[i])):
ae_models[i][j].load_state_dict(
state_dict[f'model_{i}'][j])
# push entire datasets through the (trained) classifier and trained autoencoder and save the results
for ae_model in ae_models:
if isinstance(ae_model, torch.nn.Module):
ae_model.eval()
else:
for m in ae_model:
m.eval()
# order must be preserved - alternatively use tuples and one list
loaders = [train_loader]
dataset_names = ['train']
datasets = []
for adv_type in adv_types:
loaders.append(clean_loaders[adv_type])
dataset_names.append(f'clean_{adv_type}')
loaders.append(adv_loaders[adv_type])
dataset_names.append(f'adv_{adv_type}')
loaders.append(noisy_loaders[adv_type])
dataset_names.append(f'noisy_{adv_type}')
with torch.no_grad():
for name, loader in zip(dataset_names, loaders):
filename = f'{run_name}/ae_encoded_{name}.npy'
if not Path(filename).exists():
final_features = []
with tqdm(initial=0, unit_scale=True,
dynamic_ncols=True) as pbar:
for t in loader:
if isinstance(t, torch.Tensor):
X = t
else:
X = t[0]
X = X.cuda()
y_pred, feature_list = model.feature_list(X)
features = []
for i, (feature_map, ae_model) in enumerate(
zip(feature_list, ae_models)):
feature_map = feature_map.detach()
if isinstance(ae_model, ConvVAE):
fm_recon, mu, logvar, z = ae_model.forward(
feature_map)
loss, rec_loss, div_loss = ae_model.calculate_loss(
feature_map, fm_recon, criterion, mu,
logvar)
mu_norm = torch.norm(mu, dim=1)
logvar_norm = torch.norm(logvar, dim=1)
features.append(
torch.cat(
[
rec_loss.mean(
dim=(1, 2,
3)).unsqueeze(1),
mu_norm.unsqueeze(1),
logvar_norm.unsqueeze(1)
],
dim=1).detach().cpu().numpy())
elif isinstance(ae_model, ConvWAE):
fm_recon, z = ae_model.forward(feature_map)
z_sigma = ae_model.z_var**0.5
z_gen = torch.empty(
(feature_map.size(0),
ae_model.z_dim)).normal_(
std=z_sigma).cuda()
loss, rec_loss, dis_loss = ae_model.calculate_loss(
feature_map, fm_recon, criterion,
z_gen, z)
latent_norm = torch.norm(z, dim=1)
features.append(
torch.cat(
[
rec_loss.mean(
dim=(1, 2,
3)).unsqueeze(1),
latent_norm.unsqueeze(1)
],
dim=1).detach().cpu().numpy())
elif isinstance(ae_model, tuple):
z = ae_model[0](feature_map)
fm_recon = ae_model[1](z)
dis_output = ae_model[2](z)
latent_norm = torch.norm(z, dim=1)
rec_loss = criterion(
fm_recon,
feature_map).mean(dim=(1, 2, 3))
features.append(
torch.cat(
[
rec_loss.unsqueeze(1),
latent_norm.unsqueeze(1),
dis_output
],
dim=1).detach().cpu().numpy())
else:
fm_recon, z = ae_model.forward(feature_map)
loss, rec_loss = ae_model.calculate_loss(
feature_map, fm_recon, criterion)
latent_norm = torch.norm(z, dim=1)
features.append(
torch.cat(
[
rec_loss.mean(
dim=(1, 2,
3)).unsqueeze(1),
latent_norm.unsqueeze(1)
],
dim=1).detach().cpu().numpy())
final_features.append(
np.concatenate(features, axis=1))
pbar.update()
autoencoded_set = np.concatenate(final_features, axis=0)
datasets.append(autoencoded_set)
np.save(filename, autoencoded_set)
print(f'Saving {name} encoded dataset to {filename}')
else:
print(f'Found {filename}')
datasets.append(np.load(filename))
# and save the entire latent encoding
filename = f'{run_name}/latent_{name}.npy'
if not Path(filename).exists():
final_features = []
with tqdm(initial=0, unit_scale=True,
dynamic_ncols=True) as pbar:
for t in loader:
if isinstance(t, torch.Tensor):
X = t
else:
X = t[0]
X = X.cuda()
y_pred, feature_list = model.feature_list(X)
features = []
for i, (feature_map, ae_model) in enumerate(
zip(feature_list, ae_models)):
feature_map = feature_map.detach()
if isinstance(ae_model, ConvVAE):
fm_recon, mu, logvar, z = ae_model.forward(
feature_map)
elif isinstance(ae_model, ConvWAE):
fm_recon, z = ae_model.forward(feature_map)
elif isinstance(ae_model, tuple):
z = ae_model[0](feature_map)
else:
fm_recon, z = ae_model.forward(feature_map)
features.append(z.detach().cpu().numpy())
final_features.append(
np.concatenate(features, axis=1))
pbar.update()
autoencoded_set = np.concatenate(final_features, axis=0)
datasets.append(autoencoded_set)
np.save(filename, autoencoded_set)
print(
f'Saving {name} latent encoded dataset to {filename}')
else:
print(f'Found {filename}')
# test the model performance when features are replaced by the autoencoded features
def test_ae_dataset(loader):
model.eval()
for ae_model in ae_models:
if isinstance(ae_model, torch.nn.Module):
ae_model.eval()
else:
for m in ae_model:
m.eval()
with torch.no_grad():
with tqdm(initial=0, unit_scale=True,
dynamic_ncols=True) as pbar:
total = 0
keys = ['original', 'all'
] + [i + 1 for i in range(len(ae_models))]
correct = {k: 0 for k in keys}
masks = {k: [False] * len(ae_models) for k in keys}
masks['all'] = [True] * len(ae_models)
for i in range(len(ae_models)):
masks[i + 1][i] = True
for X, y in loader:
X = X.cuda()
y = y.cuda()
total += y.size(0)
for k in keys:
y_pred = model.ae_replaced_forward(
X, ae_models, masks[k])
y_pred_max = y_pred.argmax(dim=1)
correct[k] += (y_pred_max == y).sum().item()
pbar.update()
accs = {k: correct[k] / total for k in keys}
return accs
train_accs = test_ae_dataset(train_loader)
print(f'Trainset accuracy:')
for k, v in train_accs.items():
print(f'{k}: {v}')
test_accs = test_ae_dataset(test_loader)
print(f'Testset accuracy:')
for k, v in test_accs.items():
print(f'{k}: {v}')
def main():
# set the path to pre-trained model and output
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
out_dist_list = [
'skin_cli', 'skin_derm', 'corrupted', 'corrupted_70', 'imgnet', 'nct',
'final_test'
]
# load networks
if args.net_type == 'densenet_121':
model = densenet_121.Net(models.densenet121(pretrained=False), 8)
ckpt = torch.load("../checkpoints/densenet-121/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
elif args.net_type == 'mobilenet':
model = mobilenet.Net(models.mobilenet_v2(pretrained=False), 8)
ckpt = torch.load("../checkpoints/mobilenet/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
print("Done!")
elif args.net_type == 'resnet_50':
model = resnet_50.Net(models.resnet50(pretrained=False), 8)
ckpt = torch.load("../checkpoints/resnet-50/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
print("Done!")
elif args.net_type == 'vgg_16':
model = vgg_16.Net(models.vgg16_bn(pretrained=False), 8)
ckpt = torch.load("../checkpoints/vgg-16/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
print("Done!")
else:
raise Exception(f"There is no net_type={args.net_type} available.")
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 224, 224).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print('get sample mean and covariance')
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader)
print('get Mahalanobis scores')
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print('Noise: ' + str(magnitude))
for i in range(num_output):
M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \
True, args.net_type, sample_mean, precision, i, magnitude)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
print('Out-distribution: ' + out_dist)
for i in range(num_output):
M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \
False, args.net_type, sample_mean, precision, i, magnitude)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_in)
file_name = os.path.join(
args.outf, 'Mahalanobis_%s_%s_%s.npy' %
(str(magnitude), args.dataset, out_dist))
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'malaria':
args.num_classes = 2
if args.dataset == 'cifar100':
args.num_classes = 100
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
elif args.net_type == 'resnet18':
model = models.get_model(args.net_type)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
model.cuda()
print('load model: ' + args.net_type)
# load dataset
# selected_list = torch.LongTensor(selected_list)
# test_clean_data = torch.index_select(test_clean_data, 0, selected_list)
# test_adv_data = torch.index_select(test_adv_data, 0, selected_list)
# test_noisy_data = torch.index_select(test_noisy_data, 0, selected_list)
# test_label = torch.index_select(test_label, 0, selected_list)
save_path = '%s/%s_%s/' % (args.outf, args.net_type, args.dataset)
os.makedirs(save_path, exist_ok=True)
# torch.save(test_clean_data, '%s/clean_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack))
# torch.save(test_adv_data, '%s/adv_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack))
# torch.save(test_noisy_data, '%s/noisy_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack))
# torch.save(test_label, '%s/label_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack))
print('load target data: ', args.dataset)
train_loader, _ = data_loader.getTargetDataSet(args.dataset,
args.batch_size, None,
args.dataroot)
test_clean_data = torch.load(args.outf + 'clean_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, args.adv_type))
test_adv_data = torch.load(args.outf + 'adv_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, args.adv_type))
test_noisy_data = torch.load(args.outf + 'noisy_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, args.adv_type))
test_label = torch.load(args.outf + 'label_%s_%s_%s.pth' %
(args.net_type, args.dataset, args.adv_type))
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print('get sample mean and covariance')
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader)
print('get LID scores')
LID, LID_adv, LID_noisy \
= lib_generation.get_LID(model, test_clean_data, test_adv_data, test_noisy_data, test_label, num_output)
overlap_list = [10, 20, 30, 40, 50, 60, 70, 80, 90]
list_counter = 0
for overlap in overlap_list:
Save_LID = np.asarray(LID[list_counter], dtype=np.float32)
Save_LID_adv = np.asarray(LID_adv[list_counter], dtype=np.float32)
Save_LID_noisy = np.asarray(LID_noisy[list_counter], dtype=np.float32)
Save_LID_pos = np.concatenate((Save_LID, Save_LID_noisy))
LID_data, LID_labels = lib_generation.merge_and_generate_labels(
Save_LID_adv, Save_LID_pos)
file_name = os.path.join(
args.outf,
'LID_%s_%s_%s.npy' % (overlap, args.dataset, args.adv_type))
LID_data = np.concatenate((LID_data, LID_labels), axis=1)
np.save(file_name, LID_data)
list_counter += 1
print('get Mahalanobis scores')
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print('\nNoise: ' + str(magnitude))
for i in range(num_output):
M_in \
= lib_generation.get_Mahalanobis_score_adv(model, test_clean_data, test_label, \
args.num_classes, args.outf, args.net_type, \
sample_mean, precision, i, magnitude)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for i in range(num_output):
M_out \
= lib_generation.get_Mahalanobis_score_adv(model, test_adv_data, test_label, \
args.num_classes, args.outf, args.net_type, \
sample_mean, precision, i, magnitude)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
for i in range(num_output):
M_noisy \
= lib_generation.get_Mahalanobis_score_adv(model, test_noisy_data, test_label, \
args.num_classes, args.outf, args.net_type, \
sample_mean, precision, i, magnitude)
M_noisy = np.asarray(M_noisy, dtype=np.float32)
if i == 0:
Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1))
else:
Mahalanobis_noisy = np.concatenate(
(Mahalanobis_noisy, M_noisy.reshape(
(M_noisy.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32)
Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy))
Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_pos)
file_name = os.path.join(
args.outf, 'Mahalanobis_%s_%s_%s.npy' %
(str(magnitude), args.dataset, args.adv_type))
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
default=False,
help='add center loss component')
args = parser.parse_args()
print(args)
args.cuda = not args.no_cuda and torch.cuda.is_available()
print("Random Seed: ", args.seed)
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
print('load data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, args.imageSize, args.dataroot)
print('Load model')
model = models.vgg13()
print(model)
if args.cuda:
model.cuda()
# Center loss addition
if args.centerloss:
classes = 10 # default for CIFAR & SVHN
dim = 10
centerloss = CenterLoss(num_classes=classes,
feat_dim=dim,
use_gpu=args.cuda)
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
if args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print('get Mahalanobis scores')
# m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
m_list = [0.0]
for magnitude in m_list:
print('Noise: ' + str(magnitude))
for i in range(num_output):
if args.train_flag == False:
print("OOD: True")
lib_generation.get_activations(model, test_loader, args.outf, i)
if args.train_flag == True:
print("OOD: False")
print("Extracting activations for Layer: " + str(i))
lib_generation.get_activations(model, train_loader, args.outf, i)
device = torch.device('cpu')
# set the path to pre-trained model and output
pre_trained_net = config['model_params']['pretrained_model_path']
if os.path.isdir(config['logging_params']['outf']) == False:
os.mkdir(config['logging_params']['outf'])
# create model
model = classifier_models[config['model_params']['name']](
config['exp_params']['num_classes'])
# load pretrained model
model.load_state_dict(torch.load(pre_trained_net, map_location=device))
model.to(device)
print('Loaded model: ' + config['model_params']['net_type'])
# load dataset
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
config['model_params']['transform_params']['mean'],
config['model_params']['transform_params']['std'])
])
train_loader, test_loader = data_loader.getTargetDataSet(
config['exp_params']['dataset'],
config['trainer_params']['batch_size'], in_transform,
config['exp_params']['dataroot'])
print('Loaded dataset: ', config['exp_params']['dataset'])
get_conf(config, device, model, in_transform, train_loader, test_loader)
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
for i, (name, module) in enumerate(model._modules.items()):
module = recursion_change_bn(model)
for m in model.modules():
if 'Conv' in str(type(m)):
setattr(m, 'padding_mode', 'zeros')
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, _ = data_loader.getTargetDataSet(args.dataset,
args.batch_size,
in_transform, args.dataroot)
test_clean_data = torch.load(args.outf + 'clean_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, 'FGSM'))
test_adv_data = torch.load(args.outf + 'adv_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, 'FGSM'))
test_noisy_data = torch.load(args.outf + 'noisy_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, 'FGSM'))
test_label = torch.load(args.outf + 'label_%s_%s_%s.pth' %
(args.net_type, args.dataset, 'FGSM'))
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print('get sample mean and covariance')
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader)
print('get Mahalanobis scores')
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print('\nNoise: ' + str(magnitude))
for i in range(num_output):
M_in \
= lib_generation.get_Mahalanobis_score_adv(model, test_clean_data, test_label, \
args.num_classes, args.outf, args.net_type, \
sample_mean, precision, i, magnitude)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for i in range(num_output):
M_out \
= lib_generation.get_Mahalanobis_score_adv(model, test_adv_data, test_label, \
args.num_classes, args.outf, args.net_type, \
sample_mean, precision, i, magnitude)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
for i in range(num_output):
M_noisy \
= lib_generation.get_Mahalanobis_score_adv(model, test_noisy_data, test_label, \
args.num_classes, args.outf, args.net_type, \
sample_mean, precision, i, magnitude)
M_noisy = np.asarray(M_noisy, dtype=np.float32)
if i == 0:
Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1))
else:
Mahalanobis_noisy = np.concatenate(
(Mahalanobis_noisy, M_noisy.reshape(
(M_noisy.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32)
Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy))
Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_pos)
file_name = os.path.join(
args.outf, 'Mahalanobis_%s_%s_%s.npy' %
(str(magnitude), args.dataset, 'FGSM'))
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
def main():
# set the path to pre-trained model and output
pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth"
args.outf = args.outf + args.net_type + "_" + args.dataset + "/"
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == "cifar100":
args.num_classes = 100
if args.dataset == "svhn":
out_dist_list = ["cifar10", "imagenet_resize", "lsun_resize"]
else:
out_dist_list = ["svhn", "imagenet_resize", "lsun_resize"]
# load networks
if args.net_type == "densenet":
if args.dataset == "svhn":
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))
)
else:
model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose(
[
transforms.ToTensor(),
transforms.Normalize(
(125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0),
),
]
)
elif args.net_type == "resnet":
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))
)
in_transform = transforms.Compose(
[
transforms.ToTensor(),
transforms.Normalize(
(0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)
),
]
)
model.cuda()
print("load model: " + args.net_type)
# load dataset
print("load target data: ", args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot
)
# measure the performance
M_list = [0, 0.0005, 0.001, 0.0014, 0.002, 0.0024, 0.005, 0.01, 0.05, 0.1, 0.2]
T_list = [1, 10, 100, 1000]
base_line_list = []
ODIN_best_tnr = [0, 0, 0]
ODIN_best_results = [0, 0, 0]
ODIN_best_temperature = [-1, -1, -1]
ODIN_best_magnitude = [-1, -1, -1]
for T in T_list:
for m in M_list:
magnitude = m
temperature = T
lib_generation.get_posterior(
model,
args.net_type,
test_loader,
magnitude,
temperature,
args.outf,
True,
)
out_count = 0
print("Temperature: " + str(temperature) + " / noise: " + str(magnitude))
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot
)
print("Out-distribution: " + out_dist)
lib_generation.get_posterior(
model,
args.net_type,
out_test_loader,
magnitude,
temperature,
args.outf,
False,
)
if temperature == 1 and magnitude == 0:
test_results = callog.metric(args.outf, ["PoT"])
base_line_list.append(test_results)
else:
val_results = callog.metric(args.outf, ["PoV"])
if ODIN_best_tnr[out_count] < val_results["PoV"]["TNR"]:
ODIN_best_tnr[out_count] = val_results["PoV"]["TNR"]
ODIN_best_results[out_count] = callog.metric(args.outf, ["PoT"])
ODIN_best_temperature[out_count] = temperature
ODIN_best_magnitude[out_count] = magnitude
out_count += 1
# print the results
mtypes = ["TNR", "AUROC", "DTACC", "AUIN", "AUOUT"]
print("Baseline method: in_distribution: " + args.dataset + "==========")
count_out = 0
for results in base_line_list:
print("out_distribution: " + out_dist_list[count_out])
for mtype in mtypes:
print(" {mtype:6s}".format(mtype=mtype), end="")
print("\n{val:6.2f}".format(val=100.0 * results["PoT"]["TNR"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUROC"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["DTACC"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUIN"]), end="")
print(" {val:6.2f}\n".format(val=100.0 * results["PoT"]["AUOUT"]), end="")
print("")
count_out += 1
print("ODIN method: in_distribution: " + args.dataset + "==========")
count_out = 0
for results in ODIN_best_results:
print("out_distribution: " + out_dist_list[count_out])
for mtype in mtypes:
print(" {mtype:6s}".format(mtype=mtype), end="")
print("\n{val:6.2f}".format(val=100.0 * results["PoT"]["TNR"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUROC"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["DTACC"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUIN"]), end="")
print(" {val:6.2f}\n".format(val=100.0 * results["PoT"]["AUOUT"]), end="")
print("temperature: " + str(ODIN_best_temperature[count_out]))
print("magnitude: " + str(ODIN_best_magnitude[count_out]))
print("")
count_out += 1
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
if args.dataset == 'cifar100':
args.numclass = 100
args.decreasing_lr = '80,120,160'
args.epochs = 200
train_loader, _ = data_loader.getTargetDataSet(args.dataset, args.batch_size,
transform_train, args.dataroot)
_, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size,
transform_test, args.dataroot)
if args.noise_fraction > 0:
print('load noisy labels')
args.label_root = args.label_root + args.dataset + '/' + args.noise_type + '/' + str(
args.noise_fraction)
if os.path.isdir(args.label_root) == False:
print('Err: generate noisy labels first')
else:
args.label_root = args.label_root + '/train_labels.npy'
new_label = torch.load(args.label_root)
train_loader.dataset.train_labels = new_label
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
# load networks
if args.net_type == 'densenet':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
test_clean_data, test_adv_data, test_noisy_data, test_label = {}, {}, {}, {}
clean_loaders, adv_loaders, noisy_loaders = {}, {}, {}
adv_types = ['FGSM', 'BIM', 'DeepFool', 'CWL2', 'PGD100']
for adv_type in adv_types:
test_clean_data[adv_type] = torch.load(
args.outf + 'clean_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
test_adv_data[adv_type] = torch.load(
args.outf + 'adv_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
test_noisy_data[adv_type] = torch.load(
args.outf + 'noisy_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
test_label[adv_type] = torch.load(
args.outf + 'label_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
clean_loaders[adv_type] = torch.utils.data.DataLoader(
test_clean_data[adv_type], batch_size=args.batch_size)
adv_loaders[adv_type] = torch.utils.data.DataLoader(
test_adv_data[adv_type], batch_size=args.batch_size)
noisy_loaders[adv_type] = torch.utils.data.DataLoader(
test_noisy_data[adv_type], batch_size=args.batch_size)
# nasty hack
train_clean_x = []
train_clean_y = []
for X, y in tqdm(train_loader,
desc='Aggregating train dataset',
dynamic_ncols=True):
train_clean_x.append(X.cpu().numpy())
train_clean_y.append(X.cpu().numpy())
train_clean_x = np.concatenate(train_clean_x, axis=0)
train_clean_y = np.concatenate(train_clean_y, axis=0)
X, y = train_clean_x, train_clean_y
model.eval()
# datasets = {}
method_name = 'odd_odds'
dir_name = f'{method_name}_{args.net_type}_{args.dataset}'
os.makedirs(dir_name, exist_ok=True)
# "known" variant
# results_filename = f'{dir_name}/results_known.txt'
# with open(results_filename, 'w') as results_file:
# for adv_type in adv_types:
# train_split = 0.1
# train_size = int(train_split * len(test_clean_data[adv_type]))
# test_size = len(test_clean_data[adv_type]) - train_size
# X = np.concatenate([
# test_clean_data[adv_type][:train_size],
# test_adv_data[adv_type][:train_size],
# test_noisy_data[adv_type][:train_size],
# ])
# label_y = np.concatenate([
# test_label[adv_type][:train_size],
# test_label[adv_type][:train_size],
# test_label[adv_type][:train_size],
# ])
# adv_y = np.concatenate([
# np.ones(train_size),
# np.zeros(train_size),
# np.ones(train_size),
# ])
# X_test = np.concatenate([
# test_clean_data[adv_type][train_size:],
# test_adv_data[adv_type][train_size:],
# test_noisy_data[adv_type][train_size:],
# ])
# label_y_test = np.concatenate([
# test_label[adv_type][train_size:],
# test_label[adv_type][train_size:],
# test_label[adv_type][train_size:],
# ])
# adv_y_test = np.concatenate([
# np.ones(test_size),
# np.zeros(test_size),
# np.ones(test_size),
# ])
# "unsupervised" variant
results_filename = f'{dir_name}/results_unsupervised.txt'
with open(results_filename, 'w') as results_file:
for adv_type in adv_types:
test_size = len(test_clean_data[adv_type])
X_test = np.concatenate([
test_clean_data[adv_type],
test_adv_data[adv_type],
test_noisy_data[adv_type],
])
label_y_test = np.concatenate([
test_label[adv_type],
test_label[adv_type],
test_label[adv_type],
])
adv_y_test = np.concatenate([
np.ones(test_size),
np.zeros(test_size),
np.ones(test_size),
])
adv_y_test_switched = np.concatenate([
np.zeros(test_size),
np.ones(test_size),
np.zeros(test_size),
])
# "train"
class_vectors = model.get_class_vectors()
num_classes = class_vectors.size(0)
# parameters taken from tensorflow file:
# noise_eps = 'n18.0,n24.0,n30.0'.split(',')
# noise_eps_detect = 'n30.0'.split(',')
# parameters taken from pytorch file:
noise_eps = 'n0.01,s0.01,u0.01,n0.02,s0.02,u0.02,s0.03,n0.03,u0.03'.split(
',')
noise_eps_detect = 'n0.003,s0.003,u0.003,n0.005,s0.005,u0.005,s0.008,n0.008,u0.008'.split(
',')
# noise_eps_detect = None
clip_min = min(X.min(), X_test.min())
clip_max = max(X.max(), X_test.max())
predictor = collect_statistics(
X,
y, # these are class labels, not adversarial binary labels
latent_and_logits_fn=model.forward_with_latent,
nb_classes=args.num_classes,
weights=model.get_class_vectors(),
targeted=False,
noise_eps=noise_eps,
noise_eps_detect=noise_eps_detect,
num_noise_samples=256,
batch_size=128,
clip_min=clip_min,
clip_max=clip_max,
p_ratio_cutoff=0.999,
clip_alignments=True,
cache_alignments_dir=dir_name,
)
next(predictor)
# test
y_pred_list = []
y_score_list = []
batch_size = 128
batches, remainder = divmod(X_test.shape[0], batch_size)
if remainder > 0:
batches += 1
for i in tqdm(range(batches),
dynamic_ncols=True,
desc=f"evaluating on {adv_type}"):
X_test_batch = X_test[i * batch_size:(i + 1) * batch_size]
corrected_batch, y_pred_batch, decision_batch = predictor.send(
X_test_batch).T
# print(f'corrected_batch.shape: {corrected_batch.shape}')
# print(f'y_pred_batch.shape: {y_pred_batch.shape}')
# print(f'y_pred_batch: {y_pred_batch}')
y_pred_list.append(y_pred_batch)
y_score_list.append(decision_batch)
y_pred = np.concatenate(y_pred_list, axis=0)
y_scores = np.concatenate(y_score_list, axis=0)
acc = accuracy_score(adv_y_test_switched, y_pred)
auroc = roc_auc_score(adv_y_test_switched, y_scores)
print(f'Accuracy on {adv_type}: {acc}', file=results_file)
print(f'AUROC on {adv_type}: {auroc}', file=results_file)
acc = accuracy_score(adv_y_test, y_pred)
auroc = roc_auc_score(adv_y_test, y_scores)
print(f'Accuracy (original labels) on {adv_type}: {acc}',
file=results_file)
print(f'AUROC (original labels) on {adv_type}: {auroc}',
file=results_file)
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
if args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize',] #'lsun_resize']
else:
out_dist_list = ['svhn', 'imagenet_resize',] #'lsun_resize']
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(torch.load(pre_trained_net, map_location = "cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net, map_location = "cuda:" + str(args.gpu))
in_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize([0.5071, 0.4867, 0.4408], [0.2675, 0.2565, 0.2761]),])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(torch.load(pre_trained_net, map_location = "cuda:" + str(args.gpu)))
in_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot)
# measure the performance
M_list = [0, 0.0005, 0.001, 0.0014, 0.002, 0.0024, 0.005, 0.01, 0.05, 0.1, 0.2]
T_list = [1, 10, 100, 1000]
base_line_list = []
ODIN_best_tnr = [0, 0, 0]
ODIN_best_results = [0 , 0, 0]
ODIN_best_temperature = [-1, -1, -1]
ODIN_best_magnitude = [-1, -1, -1]
for T in T_list:
for m in M_list:
magnitude = m
temperature = T
lib_generation.get_posterior(model, args.net_type, test_loader, magnitude, temperature, args.outf, True)
out_count = 0
print('Temperature: ' + str(temperature) + ' / noise: ' + str(magnitude))
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(out_dist, args.batch_size, in_transform, args.dataroot)
print('Out-distribution: ' + out_dist)
lib_generation.get_posterior(model, args.net_type, out_test_loader, magnitude, temperature, args.outf, False)
if temperature == 1 and magnitude == 0:
test_results = callog.metric(args.outf, ['PoT'])
base_line_list.append(test_results)
else:
val_results = callog.metric(args.outf, ['PoV'])
if ODIN_best_tnr[out_count] < val_results['PoV']['TNR']:
ODIN_best_tnr[out_count] = val_results['PoV']['TNR']
ODIN_best_results[out_count] = callog.metric(args.outf, ['PoT'])
ODIN_best_temperature[out_count] = temperature
ODIN_best_magnitude[out_count] = magnitude
out_count += 1
# print the results
mtypes = ['TNR', 'AUROC', 'DTACC', 'AUIN', 'AUOUT']
print('Baseline method: in_distribution: ' + args.dataset + '==========')
count_out = 0
for results in base_line_list:
print('out_distribution: '+ out_dist_list[count_out])
for mtype in mtypes:
print(' {mtype:6s}'.format(mtype=mtype), end='')
print('\n{val:6.2f}'.format(val=100.*results['PoT']['TNR']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUROC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['DTACC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUIN']), end='')
print(' {val:6.2f}\n'.format(val=100.*results['PoT']['AUOUT']), end='')
print('')
count_out += 1
print('ODIN method: in_distribution: ' + args.dataset + '==========')
count_out = 0
for results in ODIN_best_results:
print('out_distribution: '+ out_dist_list[count_out])
for mtype in mtypes:
print(' {mtype:6s}'.format(mtype=mtype), end='')
print('\n{val:6.2f}'.format(val=100.*results['PoT']['TNR']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUROC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['DTACC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUIN']), end='')
print(' {val:6.2f}\n'.format(val=100.*results['PoT']['AUOUT']), end='')
print('temperature: ' + str(ODIN_best_temperature[count_out]))
print('magnitude: '+ str(ODIN_best_magnitude[count_out]))
print('')
count_out += 1
if args.net_type == 'densenet':
num_output = 3
layer_list = list(range(num_output))
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
print('load dataset: ' + args.dataset)
num_classes = 10
if args.dataset == 'cifar100':
num_classes = 100
in_transform = transforms.Compose([transforms.ToTensor(), \
transforms.Normalize((125.3/255, 123.0/255, 113.9/255), \
(63.0/255, 62.1/255.0, 66.7/255.0)),])
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, batch_size, in_transform, args.dataroot, False)
if args.noise_fraction > 0:
print('load noisy labels')
new_label = torch.load(args.label_root)
train_loader.dataset.train_labels = new_label
num_val = 50
if args.dataset == 'cifar100':
num_val = 5
total_train_data, total_train_label, _, _ = data_loader.get_raw_data(
train_loader, num_classes, 0)
total_test_data, total_test_label, val_index, test_index \
= data_loader.get_raw_data(test_loader, num_classes, num_val)
def main():
run_dir = args.run_dir
run_name = run_dir.name
matches = runs_regex.match(run_name)
dataset = matches.group(1)
net_type = matches.group(2)
ae_type = matches.group(3)
args.num_classes = 100 if dataset == 'cifar100' else 10
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + net_type + '_' + dataset + '.pth'
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
device = f'cuda:{args.gpu}'
# load networks
if net_type == 'densenet':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print('load model: ' + net_type)
# load dataset
print('load target data: ', dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
dataset, args.batch_size, in_transform, args.dataroot)
# get the feature maps' sizes
model.eval()
temp_x = torch.rand(1, 3, 32, 32).cuda()
feature_maps = model.feature_list(temp_x)[1]
feature_map_sizes = []
for out in feature_maps:
feature_map_sizes.append(out.size()[1:])
criterion = torch.nn.CrossEntropyLoss()
ae_criterion = torch.nn.MSELoss(reduction='none')
ae_type = ae_type
filters = [128, 128, 128]
latent_size = 64
bn = False
if ae_type == 'vae':
lamb = 1e-7
elif ae_type == 'wae':
lamb = 1e-4
elif ae_type == 'waegan':
if net_type == 'densenet':
lamb = [5e-5, 1e-5, 1e-5, 1e-3]
elif net_type == 'resnet':
lamb = [5e-5, 5e-6, 5e-6, 2e-5, 5e-4]
else:
lamb = None
if ae_type == 'waegan':
ae_models = [
build_ae_for_size(ae_type, sizes, filters, latent_size, lam, bn)
for sizes, lam in zip(feature_map_sizes, lamb)
]
else:
ae_models = [
build_ae_for_size(ae_type, sizes, filters, latent_size, lamb, bn)
for sizes in feature_map_sizes
]
# read in model and autoencoder models
models_filename = f'{run_name}/models.pth'
state_dict = torch.load(models_filename)
for i, _ in enumerate(ae_models):
if isinstance(ae_models[i], torch.nn.Module):
ae_models[i].load_state_dict(state_dict[f'model_{i}'])
else:
for j in range(len(ae_models[i])):
ae_models[i][j].load_state_dict(state_dict[f'model_{i}'][j])
for ae_model in ae_models:
if isinstance(ae_model, torch.nn.Module):
ae_model.eval()
else:
for m in ae_model:
m.eval()
# read in autoencoded datasets for visualization
dataset_names = ['train']
for adv_type in adv_types:
dataset_names.append(f'clean_{adv_type}')
dataset_names.append(f'adv_{adv_type}')
dataset_names.append(f'noisy_{adv_type}')
datasets = {}
for name in dataset_names:
dataset_path = run_dir / f'ae_encoded_{name}.npy'
if dataset_path.exists():
datasets[name] = np.load(str(dataset_path))
else:
print(f'{dataset_path} is missing!')
samples_iterate = 100
test_loader_iter = iter(test_loader)
X, y = next(test_loader_iter)
X, y = X.cuda(), y.cuda()
# plot
plot_filename = f'{str(run_dir)}/adv_visualize.png'
kde_plot_filename = f'{str(run_dir)}/adv_visualize_kde.png'
labels = ['1', '34', '67', '99'] if 'densenet' in str(run_dir) else [
'1', '7', '15', '27', '33'
]
fig, ax = plt.subplots(len(ae_models),
len(adv_types),
figsize=(16 * len(adv_types), 9 * len(labels)))
kde_fig, kde_ax = plt.subplots(len(ae_models),
len(adv_types),
figsize=(16 * len(adv_types),
9 * len(labels)))
clean_samples = 250
for l, adv_type in enumerate(adv_types):
print(f'generating adv type: {adv_type}')
k = l
while True:
sample_x, sample_y = X[k].unsqueeze(0), y[k].unsqueeze(0)
# run the sample through the model
y_pred, feature_list = model.feature_list(sample_x)
# check if it is classified properly, get another sample if not
if torch.argmax(y_pred) == sample_y:
inputs = sample_x.repeat(samples_iterate, 1, 1, 1)
else:
k += 1
print(f'torch.argmax(y_pred): {torch.argmax(y_pred)}; '
f'sample_y: {sample_y}; increasing k to: {k}')
continue
# generate adversarial examples
inputs.requires_grad_()
output = model(inputs)
loss = criterion(output, y)
model.zero_grad()
loss.backward()
if net_type == 'densenet':
min_pixel = -1.98888885975
max_pixel = 2.12560367584
elif net_type == 'resnet':
min_pixel = -2.42906570435
max_pixel = 2.75373125076
if adv_type == 'FGSM':
# increasing epsilon
# adv_noise = 0.05
adv_noise = torch.linspace(0.0,
0.1,
steps=samples_iterate,
device=device).view(-1, 1, 1, 1)
gradient = torch.ge(inputs.grad.data, 0)
gradient = (gradient.float() - 0.5) * 2
if net_type == 'densenet':
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (63.0 / 255.0))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (62.1 / 255.0))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (66.7 / 255.0))
else:
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (0.2023))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (0.1994))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (0.2010))
# adv_data = torch.add(inputs.data, adv_noise, gradient)
perturbations = gradient * adv_noise
adv_data = inputs.data + perturbations
adv_data = torch.clamp(adv_data, min_pixel, max_pixel).detach()
elif adv_type == 'BIM':
# adv_noise = 0.01
adv_noise = torch.linspace(0.0,
0.02,
steps=samples_iterate,
device=device).view(-1, 1, 1, 1)
gradient = torch.sign(inputs.grad.data)
for _ in range(5):
perturbations = gradient * adv_noise
inputs = inputs + perturbations
inputs = torch.clamp(inputs, min_pixel, max_pixel).detach()
inputs.requires_grad_()
output = model(inputs)
loss = criterion(output, y)
model.zero_grad()
loss.backward()
gradient = torch.sign(inputs.grad.data)
if net_type == 'densenet':
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (63.0 / 255.0))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (62.1 / 255.0))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (66.7 / 255.0))
else:
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (0.2023))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (0.1994))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (0.2010))
perturbations = gradient * adv_noise
adv_data = inputs.data + perturbations
adv_data = torch.clamp(adv_data, min_pixel,
max_pixel).detach()
elif adv_type == 'DeepFool':
if net_type == 'resnet':
if dataset == 'cifar10':
adv_noise_base = 0.18
elif dataset == 'cifar100':
adv_noise_base = 0.03
else:
adv_noise_base = 0.1
else:
if dataset == 'cifar10':
adv_noise_base = 0.6
elif dataset == 'cifar100':
adv_noise_base = 0.1
else:
adv_noise_base = 0.5
adv_data = inputs
for i, adv_noise in enumerate(
np.linspace(0.0, adv_noise_base * 2, samples_iterate)):
sample_x_expanded = sample_x.expand(2, -1, -1, -1)
sample_y_expanded = sample_y.expand(2)
_, adv_example = adversary.deepfool(model, sample_x_expanded.data.clone(),
sample_y_expanded.data.cpu(), \
args.num_classes, step_size=adv_noise, train_mode=False)
adv_example = adv_example.cuda()
adv_data[i] = adv_example[0]
elif adv_type.startswith('PGD'):
pgd_iters = int(re.match('PGD(\d+)', adv_type).group(1))
adv_noise_base = 0.05
adv_data = inputs
for i, adv_noise in enumerate(
np.linspace(0.0, adv_noise_base * 2, samples_iterate)):
sample_x_expanded = sample_x.expand(2, -1, -1, -1)
sample_y_expanded = sample_y.expand(2)
perturbation = pgd_linf(model, sample_x_expanded,
sample_y_expanded, adv_noise, 1e-2,
100)
adv_data[i] = sample_x_expanded[0] + perturbation[0]
else:
raise ValueError(f'Unsupported adv_type value: {adv_type}')
y_adv, adv_feature_list = model.feature_list(adv_data)
if torch.argmax(y_adv[-1]) == sample_y:
k += 1
print(f'torch.argmax(y_adv[-1]): {torch.argmax(y_adv[-1])}; '
f'sample_y: {sample_y}; increasing k to: {k}')
continue
break
# get AE features for the created adv examples
features = []
for j, (ae_model,
feature_map) in enumerate(zip(ae_models, adv_feature_list)):
feature_map = feature_map.detach()
if isinstance(ae_model, ConvVAE):
fm_recon, mu, logvar, z = ae_model.forward(feature_map)
loss, rec_loss, div_loss = ae_model.calculate_loss(
feature_map, fm_recon, ae_criterion, mu, logvar)
mu_norm = torch.norm(mu, dim=1)
logvar_norm = torch.norm(logvar, dim=1)
features.append(
torch.cat([
rec_loss.mean(dim=(1, 2, 3)).unsqueeze(1),
mu_norm.unsqueeze(1),
logvar_norm.unsqueeze(1)
],
dim=1).detach().cpu().numpy())
elif isinstance(ae_model, ConvWAE):
fm_recon, z = ae_model.forward(feature_map)
z_sigma = ae_model.z_var**0.5
z_gen = torch.empty(
(feature_map.size(0),
ae_model.z_dim)).normal_(std=z_sigma).cuda()
loss, rec_loss, dis_loss = ae_model.calculate_loss(
feature_map, fm_recon, ae_criterion, z_gen, z)
latent_norm = torch.norm(z, dim=1)
features.append(
torch.cat([
rec_loss.mean(dim=(1, 2, 3)).unsqueeze(1),
latent_norm.unsqueeze(1)
],
dim=1).detach().cpu().numpy())
elif isinstance(ae_model, tuple):
z = ae_model[0](feature_map)
fm_recon = ae_model[1](z)
dis_output = ae_model[2](z)
latent_norm = torch.norm(z, dim=1)
rec_loss = ae_criterion(fm_recon,
feature_map).mean(dim=(1, 2, 3))
features.append(
torch.cat([
rec_loss.unsqueeze(1),
latent_norm.unsqueeze(1), dis_output
],
dim=1).detach().cpu().numpy())
else:
fm_recon, z = ae_model.forward(feature_map)
loss, rec_loss = ae_model.calculate_loss(
feature_map, fm_recon, ae_criterion)
latent_norm = torch.norm(z, dim=1)
features.append(
torch.cat([
rec_loss.mean(dim=(1, 2, 3)).unsqueeze(1),
latent_norm.unsqueeze(1)
],
dim=1).detach().cpu().numpy())
# generate and save figures
colors = np.zeros((adv_data.shape[0], 3))
colors[:, 1] = np.linspace(1, 0, adv_data.shape[0])
colors[:, 0] = 1
for j in range(len(features)):
# additionally plot multiple points (or KDE) from training/clean data
f1_i = j * 2
f2_i = f1_i + 1
clean_xs = datasets['train'][:, f1_i][:clean_samples]
clean_ys = datasets['train'][:, f2_i][:clean_samples]
ax[j, l].scatter(clean_xs, clean_ys, color='green')
sns.kdeplot(clean_xs,
clean_ys,
cmap='Greens',
shade=True,
shade_lowest=False,
ax=kde_ax[j, l],
alpha=1.0)
ax[j, l].scatter(features[j][:, 0],
features[j][:, 1],
color=colors)
kde_ax[j, l].scatter(features[j][:, 0],
features[j][:, 1],
color=colors)
ax[j, l].grid(True)
kde_ax[j, l].grid(True)
ax[j, l].set_title(f'{adv_type} on layer {labels[j]}')
kde_ax[j, l].set_title(f'{adv_type} on layer {labels[j]}')
ax[j, l].locator_params(nbins=5)
kde_ax[j, l].locator_params(nbins=5)
print(f'Writing {plot_filename}')
fig.savefig(plot_filename, bbox_inches='tight')
close(fig)
print(f'Writing {kde_plot_filename}')
kde_fig.savefig(kde_plot_filename, bbox_inches='tight')
close(kde_fig)
print(f'FINISHED {str(run_dir)}!')
def main():
# set the path to pre-trained model and output
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
out_dist_list = [
'skin_cli', 'skin_derm', 'corrupted', 'corrupted_70', 'imgnet', 'nct'
]
# load networks
if args.net_type == 'densenet_121':
model = densenet_121.Net(models.densenet121(pretrained=False), 8)
ckpt = torch.load("../checkpoints/densenet-121/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
elif args.net_type == 'mobilenet':
model = mobilenet.Net(models.mobilenet_v2(pretrained=False), 8)
ckpt = torch.load("../checkpoints/mobilenet/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
print("Done!")
elif args.net_type == 'resnet_50':
model = resnet_50.Net(models.resnet50(pretrained=False), 8)
ckpt = torch.load("../checkpoints/resnet-50/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
print("Done!")
elif args.net_type == 'vgg_16':
model = vgg_16.Net(models.vgg16_bn(pretrained=False), 8)
ckpt = torch.load("../checkpoints/vgg-16/checkpoint.pth")
model.load_state_dict(ckpt['model_state_dict'])
model.eval()
model.cuda()
print("Done!")
else:
raise Exception(f"There is no net_type={args.net_type} available.")
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# measure the performance
M_list = [
0, 0.0005, 0.001, 0.0014, 0.002, 0.0024, 0.005, 0.01, 0.05, 0.1, 0.2
]
T_list = [1, 10, 100, 1000]
base_line_list = []
ODIN_best_tnr = [0, 0, 0] * 2
ODIN_best_results = [0, 0, 0] * 2
ODIN_best_temperature = [-1, -1, -1] * 2
ODIN_best_magnitude = [-1, -1, -1] * 2
for T in T_list:
for m in M_list:
magnitude = m
temperature = T
lib_generation.get_posterior(model, args.net_type, test_loader,
magnitude, temperature, args.outf,
True)
out_count = 0
print('Temperature: ' + str(temperature) + ' / noise: ' +
str(magnitude))
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
print('Out-distribution: ' + out_dist)
lib_generation.get_posterior(model, args.net_type,
out_test_loader, magnitude,
temperature, args.outf, False)
if temperature == 1 and magnitude == 0:
test_results = callog.metric(args.outf, ['PoT'])
base_line_list.append(test_results)
else:
val_results = callog.metric(args.outf, ['PoV'])
if ODIN_best_tnr[out_count] < val_results['PoV']['TNR']:
ODIN_best_tnr[out_count] = val_results['PoV']['TNR']
ODIN_best_results[out_count] = callog.metric(
args.outf, ['PoT'])
ODIN_best_temperature[out_count] = temperature
ODIN_best_magnitude[out_count] = magnitude
out_count += 1
# print the results
mtypes = ['TNR', 'AUROC', 'DTACC', 'AUIN', 'AUOUT']
print('Baseline method: in_distribution: ' + args.dataset + '==========')
count_out = 0
for results in base_line_list:
print('out_distribution: ' + out_dist_list[count_out])
for mtype in mtypes:
print(' {mtype:6s}'.format(mtype=mtype), end='')
print('\n{val:6.2f}'.format(val=100. * results['PoT']['TNR']), end='')
print(' {val:6.2f}'.format(val=100. * results['PoT']['AUROC']), end='')
print(' {val:6.2f}'.format(val=100. * results['PoT']['DTACC']), end='')
print(' {val:6.2f}'.format(val=100. * results['PoT']['AUIN']), end='')
print(' {val:6.2f}\n'.format(val=100. * results['PoT']['AUOUT']),
end='')
print('')
count_out += 1
print('ODIN method: in_distribution: ' + args.dataset + '==========')
count_out = 0
for results in ODIN_best_results:
print('out_distribution: ' + out_dist_list[count_out])
for mtype in mtypes:
print(' {mtype:6s}'.format(mtype=mtype), end='')
print('\n{val:6.2f}'.format(val=100. * results['PoT']['TNR']), end='')
print(' {val:6.2f}'.format(val=100. * results['PoT']['AUROC']), end='')
print(' {val:6.2f}'.format(val=100. * results['PoT']['DTACC']), end='')
print(' {val:6.2f}'.format(val=100. * results['PoT']['AUIN']), end='')
print(' {val:6.2f}\n'.format(val=100. * results['PoT']['AUOUT']),
end='')
print('temperature: ' + str(ODIN_best_temperature[count_out]))
print('magnitude: ' + str(ODIN_best_magnitude[count_out]))
print('')
count_out += 1
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
if args.adv_type == 'FGSM':
adv_noise = 0.05
elif args.adv_type == 'BIM':
adv_noise = 0.01
elif args.adv_type == 'DeepFool':
if args.net_type == 'resnet':
if args.dataset == 'cifar10':
adv_noise = 0.18
elif args.dataset == 'cifar100':
adv_noise = 0.03
else:
adv_noise = 0.1
else:
if args.dataset == 'cifar10':
adv_noise = 0.6
elif args.dataset == 'cifar100':
adv_noise = 0.1
else:
adv_noise = 0.5
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([transforms.ToTensor(), \
transforms.Normalize((125.3/255, 123.0/255, 113.9/255), \
(63.0/255, 62.1/255.0, 66.7/255.0)),])
min_pixel = -1.98888885975
max_pixel = 2.12560367584
if args.dataset == 'cifar10':
if args.adv_type == 'FGSM':
random_noise_size = 0.21 / 4
elif args.adv_type == 'BIM':
random_noise_size = 0.21 / 4
elif args.adv_type == 'DeepFool':
random_noise_size = 0.13 * 2 / 10
elif args.adv_type == 'CWL2':
random_noise_size = 0.03 / 2
elif args.dataset == 'cifar100':
if args.adv_type == 'FGSM':
random_noise_size = 0.21 / 8
elif args.adv_type == 'BIM':
random_noise_size = 0.21 / 8
elif args.adv_type == 'DeepFool':
random_noise_size = 0.13 * 2 / 8
elif args.adv_type == 'CWL2':
random_noise_size = 0.06 / 5
else:
if args.adv_type == 'FGSM':
random_noise_size = 0.21 / 4
elif args.adv_type == 'BIM':
random_noise_size = 0.21 / 4
elif args.adv_type == 'DeepFool':
random_noise_size = 0.16 * 2 / 5
elif args.adv_type == 'CWL2':
random_noise_size = 0.07 / 2
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([transforms.ToTensor(), \
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),])
min_pixel = -2.42906570435
max_pixel = 2.75373125076
if args.dataset == 'cifar10':
if args.adv_type == 'FGSM':
random_noise_size = 0.25 / 4
elif args.adv_type == 'BIM':
random_noise_size = 0.13 / 2
elif args.adv_type == 'DeepFool':
random_noise_size = 0.25 / 4
elif args.adv_type == 'CWL2':
random_noise_size = 0.05 / 2
elif args.dataset == 'cifar100':
if args.adv_type == 'FGSM':
random_noise_size = 0.25 / 8
elif args.adv_type == 'BIM':
random_noise_size = 0.13 / 4
elif args.adv_type == 'DeepFool':
random_noise_size = 0.13 / 4
elif args.adv_type == 'CWL2':
random_noise_size = 0.05 / 2
else:
if args.adv_type == 'FGSM':
random_noise_size = 0.25 / 4
elif args.adv_type == 'BIM':
random_noise_size = 0.13 / 2
elif args.adv_type == 'DeepFool':
random_noise_size = 0.126
elif args.adv_type == 'CWL2':
random_noise_size = 0.05 / 1
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
_, test_loader = data_loader.getTargetDataSet(args.dataset,
args.batch_size,
in_transform, args.dataroot)
print('Attack: ' + args.adv_type + ', Dist: ' + args.dataset + '\n')
model.eval()
adv_data_tot, clean_data_tot, noisy_data_tot = 0, 0, 0
label_tot = 0
correct, adv_correct, noise_correct = 0, 0, 0
total, generated_noise = 0, 0
criterion = nn.CrossEntropyLoss().cuda()
selected_list = []
selected_index = 0
for data, target in test_loader:
data, target = data.cuda(), target.cuda()
data, target = Variable(data, volatile=True), Variable(target)
output = model(data)
# compute the accuracy
pred = output.data.max(1)[1]
equal_flag = pred.eq(target.data).cpu()
correct += equal_flag.sum()
noisy_data = torch.add(data.data, random_noise_size,
torch.randn(data.size()).cuda())
noisy_data = torch.clamp(noisy_data, min_pixel, max_pixel)
if total == 0:
clean_data_tot = data.clone().data.cpu()
label_tot = target.clone().data.cpu()
noisy_data_tot = noisy_data.clone().cpu()
else:
clean_data_tot = torch.cat(
(clean_data_tot, data.clone().data.cpu()), 0)
label_tot = torch.cat((label_tot, target.clone().data.cpu()), 0)
noisy_data_tot = torch.cat(
(noisy_data_tot, noisy_data.clone().cpu()), 0)
# generate adversarial
model.zero_grad()
inputs = Variable(data.data, requires_grad=True)
output = model(inputs)
loss = criterion(output, target)
loss.backward()
if args.adv_type == 'FGSM':
gradient = torch.ge(inputs.grad.data, 0)
gradient = (gradient.float() - 0.5) * 2
if args.net_type == 'densenet':
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (63.0/255.0))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (62.1/255.0))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (66.7/255.0))
else:
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (0.2023))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (0.1994))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (0.2010))
elif args.adv_type == 'BIM':
gradient = torch.sign(inputs.grad.data)
for k in range(5):
inputs = torch.add(inputs.data, adv_noise, gradient)
inputs = torch.clamp(inputs, min_pixel, max_pixel)
inputs = Variable(inputs, requires_grad=True)
output = model(inputs)
loss = criterion(output, target)
loss.backward()
gradient = torch.sign(inputs.grad.data)
if args.net_type == 'densenet':
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (63.0/255.0))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (62.1/255.0))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (66.7/255.0))
else:
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (0.2023))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (0.1994))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (0.2010))
if args.adv_type == 'DeepFool':
_, adv_data = adversary.deepfool(model, data.data.clone(), target.data.cpu(), \
args.num_classes, step_size=adv_noise, train_mode=False)
adv_data = adv_data.cuda()
elif args.adv_type == 'CWL2':
_, adv_data = adversary.cw(model,
data.data.clone(),
target.data.cpu(),
1.0,
'l2',
crop_frac=1.0)
else:
adv_data = torch.add(inputs.data, adv_noise, gradient)
adv_data = torch.clamp(adv_data, min_pixel, max_pixel)
# measure the noise
temp_noise_max = torch.abs(
(data.data - adv_data).view(adv_data.size(0), -1))
temp_noise_max, _ = torch.max(temp_noise_max, dim=1)
generated_noise += torch.sum(temp_noise_max)
if total == 0:
flag = 1
adv_data_tot = adv_data.clone().cpu()
else:
adv_data_tot = torch.cat((adv_data_tot, adv_data.clone().cpu()), 0)
output = model(Variable(adv_data, volatile=True))
# compute the accuracy
pred = output.data.max(1)[1]
equal_flag_adv = pred.eq(target.data).cpu()
adv_correct += equal_flag_adv.sum()
output = model(Variable(noisy_data, volatile=True))
# compute the accuracy
pred = output.data.max(1)[1]
equal_flag_noise = pred.eq(target.data).cpu()
noise_correct += equal_flag_noise.sum()
for i in range(data.size(0)):
if equal_flag[i] == 1 and equal_flag_noise[
i] == 1 and equal_flag_adv[i] == 0:
selected_list.append(selected_index)
selected_index += 1
total += data.size(0)
selected_list = torch.LongTensor(selected_list)
clean_data_tot = torch.index_select(clean_data_tot, 0, selected_list)
adv_data_tot = torch.index_select(adv_data_tot, 0, selected_list)
noisy_data_tot = torch.index_select(noisy_data_tot, 0, selected_list)
label_tot = torch.index_select(label_tot, 0, selected_list)
torch.save(
clean_data_tot, '%s/clean_data_%s_%s_%s.pth' %
(args.outf, args.net_type, args.dataset, args.adv_type))
torch.save(
adv_data_tot, '%s/adv_data_%s_%s_%s.pth' %
(args.outf, args.net_type, args.dataset, args.adv_type))
torch.save(
noisy_data_tot, '%s/noisy_data_%s_%s_%s.pth' %
(args.outf, args.net_type, args.dataset, args.adv_type))
torch.save(
label_tot, '%s/label_%s_%s_%s.pth' %
(args.outf, args.net_type, args.dataset, args.adv_type))
print('Adversarial Noise:({:.2f})\n'.format(generated_noise / total))
print('Final Accuracy: {}/{} ({:.2f}%)\n'.format(correct, total,
100. * correct / total))
print('Adversarial Accuracy: {}/{} ({:.2f}%)\n'.format(
adv_correct, total, 100. * adv_correct / total))
print('Noisy Accuracy: {}/{} ({:.2f}%)\n'.format(
noise_correct, total, 100. * noise_correct / total))
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
# load networks
if args.net_type == 'densenet':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
test_clean_data, test_adv_data, test_noisy_data, test_label = {}, {}, {}, {}
clean_loaders, adv_loaders, noisy_loaders = {}, {}, {}
adv_types = ['FGSM', 'BIM', 'DeepFool', 'CWL2']
for adv_type in adv_types:
test_clean_data[adv_type] = torch.load(
args.outf + 'clean_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
test_adv_data[adv_type] = torch.load(
args.outf + 'adv_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
test_noisy_data[adv_type] = torch.load(
args.outf + 'noisy_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
test_label[adv_type] = torch.load(
args.outf + 'label_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
clean_loaders[adv_type] = torch.utils.data.DataLoader(
test_clean_data[adv_type], batch_size=args.batch_size)
adv_loaders[adv_type] = torch.utils.data.DataLoader(
test_adv_data[adv_type], batch_size=args.batch_size)
noisy_loaders[adv_type] = torch.utils.data.DataLoader(
test_noisy_data[adv_type], batch_size=args.batch_size)
# get the feature maps' sizes
model.eval()
temp_x = torch.rand(1, 3, 32, 32).cuda()
feature_maps = model.feature_list(temp_x)[1]
feature_map_sizes = []
for out in feature_maps:
feature_map_sizes.append(out.size()[1:])
opt_class = torch.optim.Adam
criterion = torch.nn.MSELoss(reduction='none')
ae_type = args.ae_type
filters = [128, 128, 128]
arch_text = str(filters).replace(', ', '_').replace('(',
'').replace(')', '')
latent_size = 64
bn = False
if ae_type == 'vae':
lamb = 1e-7
elif ae_type == 'wae':
lamb = 1e-4
elif ae_type == 'waegan':
if args.net_type == 'densenet':
lamb = [5e-5, 1e-5, 1e-5, 1e-3]
elif args.net_type == 'resnet':
lamb = [5e-5, 5e-6, 5e-6, 2e-5, 5e-4]
else:
lamb = None
lr = 1e-3
for run_n in range(args.runs):
if ae_type == 'waegan':
ae_models = [
build_ae_for_size(ae_type,
sizes,
filters,
latent_size,
lam,
bn,
info=True)
for sizes, lam in zip(feature_map_sizes, lamb)
]
else:
ae_models = [
build_ae_for_size(ae_type,
sizes,
filters,
latent_size,
lamb,
bn,
info=True) for sizes in feature_map_sizes
]
run_name = (f'{args.dataset}_{args.net_type}_deep_{ae_type}_arch_{arch_text}_bn_{bn}' \
f'_latent_{latent_size}_lamb_{lamb}_lr_{lr}_bs_{args.batch_size}_epochs_{args.epochs}_{run_n}') \
.replace('.', '_').replace(',', '_')
print(f'Run name: {run_name}')
writer = SummaryWriter(run_name)
models_filename = f'{run_name}/models.pth'
models_file = Path(models_filename)
assert models_file.exists(), 'No saved model available'
state_dict = torch.load(models_filename)
for i, _ in enumerate(ae_models):
if isinstance(ae_models[i], torch.nn.Module):
ae_models[i].load_state_dict(state_dict[f'model_{i}'])
else:
for j in range(len(ae_models[i])):
ae_models[i][j].load_state_dict(
state_dict[f'model_{i}'][j])
for ae_model in ae_models:
if isinstance(ae_model, torch.nn.Module):
ae_model.eval()
else:
for m in ae_model:
m.eval()
# build reconstruction AE models
if ae_type == 'waegan':
rec_ae_models = [
build_ae_for_size(ae_type,
sizes,
filters,
latent_size,
lam,
bn,
info=True)
for sizes, lam in zip(feature_map_sizes, lamb)
]
rec_ae_opts = []
for rec_ae_model in ae_models:
rec_ae_opts.append(
tuple([
opt_class(rec_ae_model[i].parameters(), lr)
for i in range(len(rec_ae_model))
]))
else:
rec_ae_models = [
build_ae_for_size(ae_type,
sizes,
filters,
latent_size,
lamb,
bn,
info=True) for sizes in feature_map_sizes
]
rec_ae_opts = [
opt_class(rec_ae_model.parameters(), lr)
for rec_ae_model in rec_ae_models
]
# train reconstruction AE model
rec_models_filename = f'{run_name}/rec_models.pth'
rec_models_file = Path(rec_models_filename)
if not rec_models_file.exists():
current_x = 0
with tqdm(initial=current_x, unit_scale=True,
dynamic_ncols=True) as pbar:
for epoch in range(args.epochs):
for ae_model, rec_ae_model, rec_ae_opt in zip(
ae_models, rec_ae_models, rec_ae_opts):
if isinstance(ae_model, tuple):
encoder_o, decoder_o, discriminator_o = rec_ae_opt
custom_schedule(encoder_o, epoch)
custom_schedule(decoder_o, epoch)
custom_schedule(discriminator_o, epoch)
elif isinstance(ae_model, ConvWAE):
custom_schedule(rec_ae_opt, epoch)
elif isinstance(ae_model, ConvVAE):
custom_schedule(rec_ae_opt, epoch)
for X, _ in train_loader:
X = X.cuda()
y_pred, feature_list = model.feature_list(X)
for i, (feature_map, ae_model, rec_ae_model,
rec_ae_opt) in enumerate(
zip(feature_list, ae_models, rec_ae_models,
rec_ae_opts)):
feature_map = feature_map.detach()
# train on x i r(x), so:
if isinstance(ae_model, tuple):
raise NotImplementedError()
else:
if isinstance(ae_model, ConvVAE):
raise NotImplementedError()
elif isinstance(ae_model, ConvWAE):
fm_recon, _ = ae_model(feature_map)
recon_recon, z = rec_ae_model.forward(
fm_recon)
z_sigma = rec_ae_model.z_var**0.5
z_gen = torch.empty(
(fm_recon.size(0),
rec_ae_model.z_dim)).normal_(
std=z_sigma).cuda()
loss, rec_loss, dis_loss = rec_ae_model.calculate_loss(
fm_recon, recon_recon, criterion,
z_gen, z)
else:
fm_recon, _ = ae_model(feature_map)
recon_recon, z = rec_ae_model.forward(
fm_recon)
loss, rec_loss = rec_ae_model.calculate_loss(
fm_recon, recon_recon, criterion)
rec_ae_opt.zero_grad()
loss.backward()
rec_ae_opt.step()
if isinstance(ae_model, ConvWAE):
writer.add_scalar(f'Rec AE {i}/Loss',
loss.item(),
global_step=current_x)
writer.add_scalar(
f'Rec AE {i}/Reconstruction loss',
rec_loss.mean().item(),
global_step=current_x)
writer.add_scalar(
f'Rec AE {i}/Discrepancy loss',
dis_loss.item(),
global_step=current_x)
else:
writer.add_scalar(f'Rec AE {i}/Loss',
loss.item(),
global_step=current_x)
current_x += 1
pbar.set_description(f'Epoch {epoch}')
pbar.update()
# save rec AE models
state_dict = {}
for i, rec_ae_model in enumerate(rec_ae_models):
state_dict[f'model_{i}'] = rec_ae_model.state_dict()
torch.save(state_dict, rec_models_filename)
else:
state_dict = torch.load(rec_models_filename)
for i, _ in enumerate(rec_ae_models):
ae_models[i].load_state_dict(state_dict[f'model_{i}'])
# order must be preserved - alternatively use tuples and one list
loaders = [train_loader]
dataset_names = ['train']
datasets = []
for adv_type in adv_types:
loaders.append(clean_loaders[adv_type])
dataset_names.append(f'clean_{adv_type}')
loaders.append(adv_loaders[adv_type])
dataset_names.append(f'adv_{adv_type}')
loaders.append(noisy_loaders[adv_type])
dataset_names.append(f'noisy_{adv_type}')
with torch.no_grad():
for name, loader in zip(dataset_names, loaders):
filename = f'{run_name}/rec_ae_encoded_{name}.npy'
if not Path(filename).exists():
final_features = []
with tqdm(initial=0, unit_scale=True,
dynamic_ncols=True) as pbar:
for t in loader:
if isinstance(t, torch.Tensor):
X = t
else:
X = t[0]
X = X.cuda()
y_pred, feature_list = model.feature_list(X)
features = []
for i, (feature_map, ae_model,
rec_ae_model) in enumerate(
zip(feature_list, ae_models,
rec_ae_models)):
feature_map = feature_map.detach()
if isinstance(ae_model, ConvVAE):
raise NotImplementedError()
elif isinstance(ae_model, ConvWAE):
fm_recon, _ = ae_model(feature_map)
recon_recon, z = rec_ae_model.forward(
fm_recon)
z_sigma = rec_ae_model.z_var**0.5
z_gen = torch.empty(
(fm_recon.size(0),
rec_ae_model.z_dim)).normal_(
std=z_sigma).cuda()
loss, rec_loss, dis_loss = rec_ae_model.calculate_loss(
fm_recon, recon_recon, criterion,
z_gen, z)
latent_norm = torch.norm(z, dim=1)
features.append(
torch.cat(
[
rec_loss.mean(
dim=(1, 2,
3)).unsqueeze(1),
latent_norm.unsqueeze(1)
],
dim=1).detach().cpu().numpy())
elif isinstance(ae_model, tuple):
raise NotImplementedError()
else:
fm_recon, _ = ae_model(feature_map)
recon_recon, z = rec_ae_model.forward(
fm_recon)
loss, rec_loss = rec_ae_model.calculate_loss(
fm_recon, recon_recon, criterion)
latent_norm = torch.norm(z, dim=1)
features.append(
torch.cat(
[
rec_loss.mean(
dim=(1, 2,
3)).unsqueeze(1),
latent_norm.unsqueeze(1)
],
dim=1).detach().cpu().numpy())
final_features.append(
np.concatenate(features, axis=1))
pbar.update()
autoencoded_set = np.concatenate(final_features, axis=0)
datasets.append(autoencoded_set)
np.save(filename, autoencoded_set)
print(f'Saving {name} encoded dataset to {filename}')
else:
print(f'Found {filename}')
datasets.append(np.load(filename))
# and save the entire latent encoding
filename = f'{run_name}/rec_ae_latent_{name}.npy'
if not Path(filename).exists():
final_features = []
with tqdm(initial=0, unit_scale=True,
dynamic_ncols=True) as pbar:
for t in loader:
if isinstance(t, torch.Tensor):
X = t
else:
X = t[0]
X = X.cuda()
y_pred, feature_list = model.feature_list(X)
features = []
for i, (feature_map, ae_model,
rec_ae_model) in enumerate(
zip(feature_list, ae_models,
rec_ae_models)):
feature_map = feature_map.detach()
if isinstance(ae_model, ConvVAE):
raise NotImplementedError()
elif isinstance(ae_model, ConvWAE):
fm_recon, _ = ae_model(feature_map)
recon_recon, z = rec_ae_model.forward(
fm_recon)
elif isinstance(ae_model, tuple):
raise NotImplementedError()
else:
fm_recon, _ = ae_model(feature_map)
recon_recon, z = rec_ae_model.forward(
fm_recon)
features.append(z.detach().cpu().numpy())
final_features.append(
np.concatenate(features, axis=1))
pbar.update()
autoencoded_set = np.concatenate(final_features, axis=0)
datasets.append(autoencoded_set)
np.save(filename, autoencoded_set)
print(
f'Saving {name} latent encoded dataset to {filename}')
else:
print(f'Found {filename}')
