def get_model(device):
"""
:param device: instance of torch.device
:return: An instance of torch.nn.Module
"""
num_classes = 2
if config["dataset"] == "Cifar100":
num_classes = 100
elif config["dataset"] == "Cifar10":
num_classes = 10
model = {
"vgg11": lambda: models.VGG("VGG11", num_classes, batch_norm=False),
"vgg11_bn": lambda: models.VGG("VGG11", num_classes, batch_norm=True),
"vgg13": lambda: models.VGG("VGG13", num_classes, batch_norm=False),
"vgg13_bn": lambda: models.VGG("VGG13", num_classes, batch_norm=True),
"vgg16": lambda: models.VGG("VGG16", num_classes, batch_norm=False),
"vgg16_bn": lambda: models.VGG("VGG16", num_classes, batch_norm=True),
"vgg19": lambda: models.VGG("VGG19", num_classes, batch_norm=False),
"vgg19_bn": lambda: models.VGG("VGG19", num_classes, batch_norm=True),
"resnet10": lambda: models.ResNet10(num_classes=num_classes),
"resnet18": lambda: models.ResNet18(num_classes=num_classes),
"resnet34": lambda: models.ResNet34(num_classes=num_classes),
"resnet50": lambda: models.ResNet50(num_classes=num_classes),
"resnet101": lambda: models.ResNet101(num_classes=num_classes),
"resnet152": lambda: models.ResNet152(num_classes=num_classes),
"bert": lambda: models.BertImage(config, num_classes=num_classes),
}[config["model"]]()
model.to(device)
if device == "cuda":
model = torch.nn.DataParallel(model)
torch.backends.cudnn.benchmark = True
return model
def main():
"""
Args:
- dataset: name of dataset to attack (expected cifar10/cifar100/svhn)
- attack: type of attack to launch (expected fgsm | deepfool | bim | cwl2)
- batch_size: batch size for data loader
- gpu: gpu index
"""
# load data
num_classes = 100 if args.dataset == 'cifar100' else 10
_, loader = data_loader.getTargetDataSet(args.dataset, args.batch_size,
args.data_path)
# load model
model = models.ResNet34(num_c=num_classes)
model.load_state_dict(
torch.load(NET_PATH, map_location="cuda:" + str(args.gpu)))
model.cuda()
# apply attack
if args.attack == "all":
for attack in ADVERSARIAL:
applyAttack(attack, model, loader, num_classes)
else:
applyAttack(args.attack, model, loader, num_classes)
def __init__(self, in_data, net_path, save_path, batch_size=args.batch_size):
"""
Args:
- in_data: name of in-dataset
- net_path: path to pre-trained net weights (corresponding to in_data)
- save_path: output file path, must exist
- batch_size: batch size for data loader (both in/out)
"""
self.in_data = in_data
self.num_classes = 100 if self.in_data == 'cifar100' else 10
self.batch_size = batch_size
self.save_path = save_path
# load training data
self.train_loader, self.in_test_loader = data_loader.getTargetDataSet(
self.in_data, self.batch_size, args.data_path)
# load model
self.model = models.ResNet34(num_c=self.num_classes)
self.model.load_state_dict(torch.load(net_path, map_location = "cuda:" + str(args.gpu)))
self.model.cuda()
# get information about num layers, activation sizes by trying a test input
self.model.eval()
temp_x = Variable(torch.rand(2, 3, 32, 32).cuda())
temp_list = self.model.feature_list(temp_x)[1]
self.num_layers = len(temp_list)
self.activation_sizes = [layer.size(1) for layer in temp_list]
def test_attributes(params):
model = models.ResNet34(pretrained=False)
trainer = trainers.BengaliTrainer(model=model)
bengali = engines.BengaliEngine(trainer=trainer, params=params)
assert isinstance(bengali.trainer, trainers.BaseTrainer)
assert issubclass(bengali.training_constructor,
datasets.BengaliDataSetTrain)
assert issubclass(bengali.val_constructor, datasets.BengaliDataSetTrain)
assert issubclass(bengali.test_constructor, datasets.BengaliDataSetTest)
def model_dispatcher(base_model):
if base_model == 'se_resnext101_32x4d':
return models.SE_ResNext101_32x4d(pretrained=True, n_class=3)
elif base_model == 'vgg16':
return models.VGG16(pretrained=True, n_class=3)
elif base_model == 'resnet34':
return models.ResNet34(pretrained=True, n_class=3)
elif base_model == 'se_resnext101_32x4d_sSE':
return models.se_resnext101_32x4d_sSE(pretrained=True, n_class=3)
def model_dispatcher(if_pretrain, base_model, nclass):
if base_model == 'se_resnext101_32x4d':
return models.SE_ResNext101_32x4d(pretrained=if_pretrain,
n_class=nclass)
elif base_model == 'vgg16':
return models.VGG16(pretrained=if_pretrain, n_class=nclass)
elif base_model == 'resnet34':
return models.ResNet34(pretrained=if_pretrain, n_class=nclass)
elif base_model == 'se_resnext101_32x4d_sSE':
return models.se_resnext101_32x4d_sSE(pretrained=if_pretrain,
n_class=nclass)
elif base_model == 'EfficientNet_B6':
return models.EfficientNet_B6(pretrained=if_pretrain, n_class=nclass)
def main():
# set the path to pre-trained model and output
pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth"
args.outf = args.outf + args.net_type + "_" + args.dataset + "/"
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == "cifar100":
args.num_classes = 100
if args.dataset == "svhn":
out_dist_list = ["cifar10", "imagenet_resize", "lsun_resize"]
else:
out_dist_list = ["svhn", "imagenet_resize", "lsun_resize"]
# load networks
if args.net_type == "densenet":
if args.dataset == "svhn":
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
(125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0),
),
])
elif args.net_type == "resnet":
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print("load model: " + args.net_type)
# load dataset
print("load target data: ", args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print("get sample mean and covariance")
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader)
print("get Mahalanobis scores")
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print("Noise: " + str(magnitude))
for i in range(num_output):
M_in = lib_generation.get_Mahalanobis_score(
model,
test_loader,
args.num_classes,
args.outf,
True,
args.net_type,
sample_mean,
precision,
i,
magnitude,
)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
print("Out-distribution: " + out_dist)
for i in range(num_output):
M_out = lib_generation.get_Mahalanobis_score(
model,
out_test_loader,
args.num_classes,
args.outf,
False,
args.net_type,
sample_mean,
precision,
i,
magnitude,
)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
(
Mahalanobis_data,
Mahalanobis_labels,
) = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_in)
file_name = os.path.join(
args.outf,
"Mahalanobis_%s_%s_%s.npy" %
(str(magnitude), args.dataset, out_dist),
)
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'malaria':
args.num_classes = 2
if args.dataset == 'cifar100':
args.num_classes = 100
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
elif args.net_type == 'resnet18':
model = models.get_model(args.net_type)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
model.cuda()
print('load model: ' + args.net_type)
# load dataset
# selected_list = torch.LongTensor(selected_list)
# test_clean_data = torch.index_select(test_clean_data, 0, selected_list)
# test_adv_data = torch.index_select(test_adv_data, 0, selected_list)
# test_noisy_data = torch.index_select(test_noisy_data, 0, selected_list)
# test_label = torch.index_select(test_label, 0, selected_list)
save_path = '%s/%s_%s/' % (args.outf, args.net_type, args.dataset)
os.makedirs(save_path, exist_ok=True)
# torch.save(test_clean_data, '%s/clean_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack))
# torch.save(test_adv_data, '%s/adv_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack))
# torch.save(test_noisy_data, '%s/noisy_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack))
# torch.save(test_label, '%s/label_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack))
print('load target data: ', args.dataset)
train_loader, _ = data_loader.getTargetDataSet(args.dataset,
args.batch_size, None,
args.dataroot)
test_clean_data = torch.load(args.outf + 'clean_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, args.adv_type))
test_adv_data = torch.load(args.outf + 'adv_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, args.adv_type))
test_noisy_data = torch.load(args.outf + 'noisy_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, args.adv_type))
test_label = torch.load(args.outf + 'label_%s_%s_%s.pth' %
(args.net_type, args.dataset, args.adv_type))
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print('get sample mean and covariance')
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader)
print('get LID scores')
LID, LID_adv, LID_noisy \
= lib_generation.get_LID(model, test_clean_data, test_adv_data, test_noisy_data, test_label, num_output)
overlap_list = [10, 20, 30, 40, 50, 60, 70, 80, 90]
list_counter = 0
for overlap in overlap_list:
Save_LID = np.asarray(LID[list_counter], dtype=np.float32)
Save_LID_adv = np.asarray(LID_adv[list_counter], dtype=np.float32)
Save_LID_noisy = np.asarray(LID_noisy[list_counter], dtype=np.float32)
Save_LID_pos = np.concatenate((Save_LID, Save_LID_noisy))
LID_data, LID_labels = lib_generation.merge_and_generate_labels(
Save_LID_adv, Save_LID_pos)
file_name = os.path.join(
args.outf,
'LID_%s_%s_%s.npy' % (overlap, args.dataset, args.adv_type))
LID_data = np.concatenate((LID_data, LID_labels), axis=1)
np.save(file_name, LID_data)
list_counter += 1
print('get Mahalanobis scores')
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print('\nNoise: ' + str(magnitude))
for i in range(num_output):
M_in \
= lib_generation.get_Mahalanobis_score_adv(model, test_clean_data, test_label, \
args.num_classes, args.outf, args.net_type, \
sample_mean, precision, i, magnitude)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for i in range(num_output):
M_out \
= lib_generation.get_Mahalanobis_score_adv(model, test_adv_data, test_label, \
args.num_classes, args.outf, args.net_type, \
sample_mean, precision, i, magnitude)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
for i in range(num_output):
M_noisy \
= lib_generation.get_Mahalanobis_score_adv(model, test_noisy_data, test_label, \
args.num_classes, args.outf, args.net_type, \
sample_mean, precision, i, magnitude)
M_noisy = np.asarray(M_noisy, dtype=np.float32)
if i == 0:
Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1))
else:
Mahalanobis_noisy = np.concatenate(
(Mahalanobis_noisy, M_noisy.reshape(
(M_noisy.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32)
Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy))
Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_pos)
file_name = os.path.join(
args.outf, 'Mahalanobis_%s_%s_%s.npy' %
(str(magnitude), args.dataset, args.adv_type))
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
if args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print('get Mahalanobis scores')
# m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
m_list = [0.0]
for magnitude in m_list:
print('Noise: ' + str(magnitude))
for i in range(num_output):
if args.train_flag == False:
print("OOD: True")
lib_generation.get_activations(model, test_loader, args.outf, i)
if args.train_flag == True:
print("OOD: False")
print("Extracting activations for Layer: " + str(i))
lib_generation.get_activations(model, train_loader, args.outf, i)
def main():
# set the path to pre-trained model and output
pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth"
args.outf = args.outf + args.net_type + "_" + args.dataset + "/"
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == "cifar100":
args.num_classes = 100
# load networks
if args.net_type == "densenet":
if args.dataset == "svhn":
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
(125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0),
),
])
elif args.net_type == "resnet":
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print("load model: " + args.net_type)
# load dataset
print("load target data: ", args.dataset)
train_loader, _ = data_loader.getTargetDataSet(args.dataset,
args.batch_size,
in_transform, args.dataroot)
test_clean_data = torch.load(args.outf + "clean_data_%s_%s_%s.pth" %
(args.net_type, args.dataset, args.adv_type))
test_adv_data = torch.load(args.outf + "adv_data_%s_%s_%s.pth" %
(args.net_type, args.dataset, args.adv_type))
test_noisy_data = torch.load(args.outf + "noisy_data_%s_%s_%s.pth" %
(args.net_type, args.dataset, args.adv_type))
test_label = torch.load(args.outf + "label_%s_%s_%s.pth" %
(args.net_type, args.dataset, args.adv_type))
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print("get sample mean and covariance")
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader)
print("get LID scores")
LID, LID_adv, LID_noisy = lib_generation.get_LID(model, test_clean_data,
test_adv_data,
test_noisy_data,
test_label, num_output)
overlap_list = [10, 20, 30, 40, 50, 60, 70, 80, 90]
list_counter = 0
for overlap in overlap_list:
Save_LID = np.asarray(LID[list_counter], dtype=np.float32)
Save_LID_adv = np.asarray(LID_adv[list_counter], dtype=np.float32)
Save_LID_noisy = np.asarray(LID_noisy[list_counter], dtype=np.float32)
Save_LID_pos = np.concatenate((Save_LID, Save_LID_noisy))
LID_data, LID_labels = lib_generation.merge_and_generate_labels(
Save_LID_adv, Save_LID_pos)
file_name = os.path.join(
args.outf,
"LID_%s_%s_%s.npy" % (overlap, args.dataset, args.adv_type))
LID_data = np.concatenate((LID_data, LID_labels), axis=1)
np.save(file_name, LID_data)
list_counter += 1
print("get Mahalanobis scores")
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print("\nNoise: " + str(magnitude))
for i in range(num_output):
M_in = lib_generation.get_Mahalanobis_score_adv(
model,
test_clean_data,
test_label,
args.num_classes,
args.outf,
args.net_type,
sample_mean,
precision,
i,
magnitude,
)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for i in range(num_output):
M_out = lib_generation.get_Mahalanobis_score_adv(
model,
test_adv_data,
test_label,
args.num_classes,
args.outf,
args.net_type,
sample_mean,
precision,
i,
magnitude,
)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
for i in range(num_output):
M_noisy = lib_generation.get_Mahalanobis_score_adv(
model,
test_noisy_data,
test_label,
args.num_classes,
args.outf,
args.net_type,
sample_mean,
precision,
i,
magnitude,
)
M_noisy = np.asarray(M_noisy, dtype=np.float32)
if i == 0:
Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1))
else:
Mahalanobis_noisy = np.concatenate(
(Mahalanobis_noisy, M_noisy.reshape(
(M_noisy.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32)
Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy))
Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_pos)
file_name = os.path.join(
args.outf,
"Mahalanobis_%s_%s_%s.npy" %
(str(magnitude), args.dataset, args.adv_type),
)
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
def main():
if os.path.isdir('feature_lists') == False:
os.mkdir('feature_lists')
if args.dataset == 'cifar100':
args.num_classes = 100
else:
args.num_classes = 10
# load networks
pre_trained_net = args.net_type + '_' + args.dataset + '.pth'
pre_trained_net = os.path.join('pre_trained', pre_trained_net)
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(0)))
else:
model = torch.load(pre_trained_net, map_location="cpu")
for i, (name, module) in enumerate(model._modules.items()):
module = recursion_change_bn(model)
for m in model.modules():
if 'Conv' in str(type(m)):
setattr(m, 'padding_mode', 'zeros')
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(0)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
if args.validation_src == 'FGSM':
if args.dataset == 'svhn':
out_dist_list = [
'cifar10', 'imagenet_resize', 'lsun_resize', 'FGSM'
]
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize', 'FGSM']
else:
if args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
print('load model: ' + args.net_type)
model.to(device)
model.eval()
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extraction
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
correct, total = 0, 0
num_output = len(feature_list)
num_sample_per_class = np.empty(args.num_classes)
num_sample_per_class.fill(0)
list_features = []
list_features_test = []
list_features_out = []
for i in range(num_output):
temp_list = []
list_features_test.append(0)
list_features_out.append(0)
for j in range(args.num_classes):
temp_list.append(0)
list_features.append(temp_list)
for data, target in train_loader:
total += data.size(0)
data = data.cuda()
data = Variable(data, volatile=True)
output, out_features = model.feature_list(data)
# get hidden features
for i in range(num_output):
out_features[i] = out_features[i].view(out_features[i].size(0),
out_features[i].size(1), -1)
out_features[i] = torch.mean(out_features[i].data, 2)
# compute the accuracy
pred = output.data.max(1)[1]
equal_flag = pred.eq(target.cuda()).cpu()
correct += equal_flag.sum()
# construct the sample matrix
for i in range(data.size(0)):
label = target[i]
if num_sample_per_class[label] == 0:
out_count = 0
for out in out_features:
list_features[out_count][label] = out[i].view(1, -1)
out_count += 1
else:
out_count = 0
for out in out_features:
list_features[out_count][label] = torch.cat(
(list_features[out_count][label], out[i].view(1, -1)),
0)
out_count += 1
num_sample_per_class[label] += 1
sample_class_mean = []
out_count = 0
for num_feature in feature_list:
temp_list = torch.Tensor(args.num_classes, int(num_feature)).cuda()
for j in range(args.num_classes):
temp_list[j] = torch.mean(list_features[out_count][j], 0)
sample_class_mean.append(temp_list)
out_count += 1
A = []
A_inv = []
log_abs_det_A_inv = []
for k in range(num_output):
X = 0
for i in range(args.num_classes):
if i == 0:
X = list_features[k][i] - sample_class_mean[k][i]
else:
X = torch.cat(
(X, list_features[k][i] - sample_class_mean[k][i]), 0)
# find inverse
u, s, vh = np.linalg.svd((X.cpu().numpy()) / np.sqrt(X.shape[0]),
full_matrices=False)
covariance_real = np.cov(X.cpu().numpy().T)
valid_indx = s > 1e-5
if (valid_indx.sum() % 2 > 0):
valid_indx[valid_indx.sum() - 1] = False
covriance_cal = np.matmul(
np.matmul(vh[valid_indx, :].transpose(),
np.diag(s[valid_indx]**2)), vh[valid_indx, :])
A_temp = np.matmul(vh[valid_indx, :].transpose(),
np.diag(s[valid_indx]))
A.append(A_temp)
covriance_cal2 = np.matmul(A_temp, A_temp.transpose())
s_inv = 1 / s[valid_indx]
A_inv_temp = np.matmul(np.diag(s_inv), vh[valid_indx, :])
A_inv.append(A_inv_temp)
log_abs_det_A_inv_temp = np.sum(np.log(np.abs(s_inv)))
log_abs_det_A_inv.append(log_abs_det_A_inv_temp)
print('\n Training Accuracy:({:.2f}%)\n'.format(100.0 * int(correct) /
int(total)))
num_sample_per_output = np.empty(num_output)
num_sample_per_output.fill(0)
for data, target in test_loader:
data = data.cuda()
data = Variable(data, volatile=True)
output, out_features = model.feature_list(data)
# get hidden features
for i in range(num_output):
out_features[i] = out_features[i].view(out_features[i].size(0),
out_features[i].size(1), -1)
out_features[i] = torch.mean(out_features[i].data, 2)
if num_sample_per_output[i] == 0:
list_features_test[i] = out_features[i]
else:
list_features_test[i] = torch.cat(
(list_features_test[i], out_features[i]), 0)
num_sample_per_output[i] += 1
for out_dist in out_dist_list:
if out_dist == 'FGSM':
test_loader, out_test_loader = data_loader.getFGSM(
args.batch_size, args.dataset, args.net_type)
num_sample_per_output.fill(0)
for data in test_loader:
data = data.cuda()
data = Variable(data, volatile=True)
output, out_features = model.feature_list(data)
# get hidden features
for i in range(num_output):
out_features[i] = out_features[i].view(
out_features[i].size(0), out_features[i].size(1), -1)
out_features[i] = torch.mean(out_features[i].data, 2)
if num_sample_per_output[i] == 0:
list_features_test[i] = out_features[i]
else:
list_features_test[i] = torch.cat(
(list_features_test[i], out_features[i]), 0)
num_sample_per_output[i] += 1
num_sample_per_output = np.empty(num_output)
num_sample_per_output.fill(0)
for data in out_test_loader:
data = data.cuda()
data = Variable(data, requires_grad=True)
output, out_features = model.feature_list(data)
# get hidden features
for i in range(num_output):
out_features[i] = out_features[i].view(
out_features[i].size(0), out_features[i].size(1), -1)
out_features[i] = torch.mean(out_features[i].data, 2)
if num_sample_per_output[i] == 0:
list_features_out[i] = out_features[i]
else:
list_features_out[i] = torch.cat(
(list_features_out[i], out_features[i]), 0)
num_sample_per_output[i] += 1
for i in range(num_output):
sample_class_mean[i] = sample_class_mean[i].cpu()
list_features_test[i] = list_features_test[i].cpu()
list_features_out[i] = list_features_out[i].cpu()
for j in range(args.num_classes):
list_features[i][j] = list_features[i][j].cpu()
else:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
num_sample_per_output.fill(0)
for data, target in out_test_loader:
data, target = data.cuda(), target.cuda()
data, target = Variable(data,
requires_grad=True), Variable(target)
output, out_features = model.feature_list(data)
# get hidden features
for i in range(num_output):
out_features[i] = out_features[i].view(
out_features[i].size(0), out_features[i].size(1), -1)
out_features[i] = torch.mean(out_features[i].data, 2)
if num_sample_per_output[i] == 0:
list_features_out[i] = out_features[i]
else:
list_features_out[i] = torch.cat(
(list_features_out[i], out_features[i]), 0)
num_sample_per_output[i] += 1
for i in range(num_output):
sample_class_mean[i] = sample_class_mean[i].cpu()
list_features_test[i] = list_features_test[i].cpu()
list_features_out[i] = list_features_out[i].cpu()
for j in range(args.num_classes):
list_features[i][j] = list_features[i][j].cpu()
file_name = os.path.join(
'feature_lists', 'feature_lists_{}_{}_{}_Wlinear.pickle'.format(
args.net_type, out_dist, args.dataset))
with open(file_name, 'wb') as f:
pickle.dump([
sample_class_mean, list_features, list_features_test,
list_features_out, A, A_inv, log_abs_det_A_inv
], f)
print('>>>>> Load Data')
# change to dataset
train_dataset = Data.TensorDataset(train_x, train_y)
test_dataset = Data.TensorDataset(test_x, test_y)
# load data
loader_train = Data.DataLoader(
dataset=train_dataset, # torch TensorDataset format
batch_size=Args.batch_size, # mini batch size
shuffle=True, # shuffle or not
num_workers=0, # multi-worker for load data
)
loader_test = Data.DataLoader(test_dataset, Args.batch_size)
del ECG_data, ECG_label, train_x, test_x, train_y, test_y
# %% ########## Create model ##########
print('>>>>> Create model')
resnet = models.ResNet34().cuda() if Args.cuda else models.ResNet34()
# optimizer
optimizer = torch.optim.Adam(resnet.parameters(), lr=Args.learn_rate)
# loss function
loss_func = nn.CrossEntropyLoss()
# evaluate
Accuracy = []
F1 = []
# save
Best_result = [0, [], []]
# %% ########## Training ##########
if Args.show_plot:
fig = plt.figure(1)
plt.ion()
print('>>>>> Start Training')
for epoch in range(Args.epoch):
def main():
run_dir = args.run_dir
run_name = run_dir.name
matches = runs_regex.match(run_name)
dataset = matches.group(1)
net_type = matches.group(2)
ae_type = matches.group(3)
args.num_classes = 100 if dataset == 'cifar100' else 10
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + net_type + '_' + dataset + '.pth'
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
device = f'cuda:{args.gpu}'
# load networks
if net_type == 'densenet':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print('load model: ' + net_type)
# load dataset
print('load target data: ', dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
dataset, args.batch_size, in_transform, args.dataroot)
# get the feature maps' sizes
model.eval()
temp_x = torch.rand(1, 3, 32, 32).cuda()
feature_maps = model.feature_list(temp_x)[1]
feature_map_sizes = []
for out in feature_maps:
feature_map_sizes.append(out.size()[1:])
criterion = torch.nn.CrossEntropyLoss()
ae_criterion = torch.nn.MSELoss(reduction='none')
ae_type = ae_type
filters = [128, 128, 128]
latent_size = 64
bn = False
if ae_type == 'vae':
lamb = 1e-7
elif ae_type == 'wae':
lamb = 1e-4
elif ae_type == 'waegan':
if net_type == 'densenet':
lamb = [5e-5, 1e-5, 1e-5, 1e-3]
elif net_type == 'resnet':
lamb = [5e-5, 5e-6, 5e-6, 2e-5, 5e-4]
else:
lamb = None
if ae_type == 'waegan':
ae_models = [
build_ae_for_size(ae_type, sizes, filters, latent_size, lam, bn)
for sizes, lam in zip(feature_map_sizes, lamb)
]
else:
ae_models = [
build_ae_for_size(ae_type, sizes, filters, latent_size, lamb, bn)
for sizes in feature_map_sizes
]
# read in model and autoencoder models
models_filename = f'{run_name}/models.pth'
state_dict = torch.load(models_filename)
for i, _ in enumerate(ae_models):
if isinstance(ae_models[i], torch.nn.Module):
ae_models[i].load_state_dict(state_dict[f'model_{i}'])
else:
for j in range(len(ae_models[i])):
ae_models[i][j].load_state_dict(state_dict[f'model_{i}'][j])
for ae_model in ae_models:
if isinstance(ae_model, torch.nn.Module):
ae_model.eval()
else:
for m in ae_model:
m.eval()
# read in autoencoded datasets for visualization
dataset_names = ['train']
for adv_type in adv_types:
dataset_names.append(f'clean_{adv_type}')
dataset_names.append(f'adv_{adv_type}')
dataset_names.append(f'noisy_{adv_type}')
datasets = {}
for name in dataset_names:
dataset_path = run_dir / f'ae_encoded_{name}.npy'
if dataset_path.exists():
datasets[name] = np.load(str(dataset_path))
else:
print(f'{dataset_path} is missing!')
samples_iterate = 100
test_loader_iter = iter(test_loader)
X, y = next(test_loader_iter)
X, y = X.cuda(), y.cuda()
# plot
plot_filename = f'{str(run_dir)}/adv_visualize.png'
kde_plot_filename = f'{str(run_dir)}/adv_visualize_kde.png'
labels = ['1', '34', '67', '99'] if 'densenet' in str(run_dir) else [
'1', '7', '15', '27', '33'
]
fig, ax = plt.subplots(len(ae_models),
len(adv_types),
figsize=(16 * len(adv_types), 9 * len(labels)))
kde_fig, kde_ax = plt.subplots(len(ae_models),
len(adv_types),
figsize=(16 * len(adv_types),
9 * len(labels)))
clean_samples = 250
for l, adv_type in enumerate(adv_types):
print(f'generating adv type: {adv_type}')
k = l
while True:
sample_x, sample_y = X[k].unsqueeze(0), y[k].unsqueeze(0)
# run the sample through the model
y_pred, feature_list = model.feature_list(sample_x)
# check if it is classified properly, get another sample if not
if torch.argmax(y_pred) == sample_y:
inputs = sample_x.repeat(samples_iterate, 1, 1, 1)
else:
k += 1
print(f'torch.argmax(y_pred): {torch.argmax(y_pred)}; '
f'sample_y: {sample_y}; increasing k to: {k}')
continue
# generate adversarial examples
inputs.requires_grad_()
output = model(inputs)
loss = criterion(output, y)
model.zero_grad()
loss.backward()
if net_type == 'densenet':
min_pixel = -1.98888885975
max_pixel = 2.12560367584
elif net_type == 'resnet':
min_pixel = -2.42906570435
max_pixel = 2.75373125076
if adv_type == 'FGSM':
# increasing epsilon
# adv_noise = 0.05
adv_noise = torch.linspace(0.0,
0.1,
steps=samples_iterate,
device=device).view(-1, 1, 1, 1)
gradient = torch.ge(inputs.grad.data, 0)
gradient = (gradient.float() - 0.5) * 2
if net_type == 'densenet':
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (63.0 / 255.0))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (62.1 / 255.0))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (66.7 / 255.0))
else:
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (0.2023))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (0.1994))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (0.2010))
# adv_data = torch.add(inputs.data, adv_noise, gradient)
perturbations = gradient * adv_noise
adv_data = inputs.data + perturbations
adv_data = torch.clamp(adv_data, min_pixel, max_pixel).detach()
elif adv_type == 'BIM':
# adv_noise = 0.01
adv_noise = torch.linspace(0.0,
0.02,
steps=samples_iterate,
device=device).view(-1, 1, 1, 1)
gradient = torch.sign(inputs.grad.data)
for _ in range(5):
perturbations = gradient * adv_noise
inputs = inputs + perturbations
inputs = torch.clamp(inputs, min_pixel, max_pixel).detach()
inputs.requires_grad_()
output = model(inputs)
loss = criterion(output, y)
model.zero_grad()
loss.backward()
gradient = torch.sign(inputs.grad.data)
if net_type == 'densenet':
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (63.0 / 255.0))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (62.1 / 255.0))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (66.7 / 255.0))
else:
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (0.2023))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (0.1994))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (0.2010))
perturbations = gradient * adv_noise
adv_data = inputs.data + perturbations
adv_data = torch.clamp(adv_data, min_pixel,
max_pixel).detach()
elif adv_type == 'DeepFool':
if net_type == 'resnet':
if dataset == 'cifar10':
adv_noise_base = 0.18
elif dataset == 'cifar100':
adv_noise_base = 0.03
else:
adv_noise_base = 0.1
else:
if dataset == 'cifar10':
adv_noise_base = 0.6
elif dataset == 'cifar100':
adv_noise_base = 0.1
else:
adv_noise_base = 0.5
adv_data = inputs
for i, adv_noise in enumerate(
np.linspace(0.0, adv_noise_base * 2, samples_iterate)):
sample_x_expanded = sample_x.expand(2, -1, -1, -1)
sample_y_expanded = sample_y.expand(2)
_, adv_example = adversary.deepfool(model, sample_x_expanded.data.clone(),
sample_y_expanded.data.cpu(), \
args.num_classes, step_size=adv_noise, train_mode=False)
adv_example = adv_example.cuda()
adv_data[i] = adv_example[0]
elif adv_type.startswith('PGD'):
pgd_iters = int(re.match('PGD(\d+)', adv_type).group(1))
adv_noise_base = 0.05
adv_data = inputs
for i, adv_noise in enumerate(
np.linspace(0.0, adv_noise_base * 2, samples_iterate)):
sample_x_expanded = sample_x.expand(2, -1, -1, -1)
sample_y_expanded = sample_y.expand(2)
perturbation = pgd_linf(model, sample_x_expanded,
sample_y_expanded, adv_noise, 1e-2,
100)
adv_data[i] = sample_x_expanded[0] + perturbation[0]
else:
raise ValueError(f'Unsupported adv_type value: {adv_type}')
y_adv, adv_feature_list = model.feature_list(adv_data)
if torch.argmax(y_adv[-1]) == sample_y:
k += 1
print(f'torch.argmax(y_adv[-1]): {torch.argmax(y_adv[-1])}; '
f'sample_y: {sample_y}; increasing k to: {k}')
continue
break
# get AE features for the created adv examples
features = []
for j, (ae_model,
feature_map) in enumerate(zip(ae_models, adv_feature_list)):
feature_map = feature_map.detach()
if isinstance(ae_model, ConvVAE):
fm_recon, mu, logvar, z = ae_model.forward(feature_map)
loss, rec_loss, div_loss = ae_model.calculate_loss(
feature_map, fm_recon, ae_criterion, mu, logvar)
mu_norm = torch.norm(mu, dim=1)
logvar_norm = torch.norm(logvar, dim=1)
features.append(
torch.cat([
rec_loss.mean(dim=(1, 2, 3)).unsqueeze(1),
mu_norm.unsqueeze(1),
logvar_norm.unsqueeze(1)
],
dim=1).detach().cpu().numpy())
elif isinstance(ae_model, ConvWAE):
fm_recon, z = ae_model.forward(feature_map)
z_sigma = ae_model.z_var**0.5
z_gen = torch.empty(
(feature_map.size(0),
ae_model.z_dim)).normal_(std=z_sigma).cuda()
loss, rec_loss, dis_loss = ae_model.calculate_loss(
feature_map, fm_recon, ae_criterion, z_gen, z)
latent_norm = torch.norm(z, dim=1)
features.append(
torch.cat([
rec_loss.mean(dim=(1, 2, 3)).unsqueeze(1),
latent_norm.unsqueeze(1)
],
dim=1).detach().cpu().numpy())
elif isinstance(ae_model, tuple):
z = ae_model[0](feature_map)
fm_recon = ae_model[1](z)
dis_output = ae_model[2](z)
latent_norm = torch.norm(z, dim=1)
rec_loss = ae_criterion(fm_recon,
feature_map).mean(dim=(1, 2, 3))
features.append(
torch.cat([
rec_loss.unsqueeze(1),
latent_norm.unsqueeze(1), dis_output
],
dim=1).detach().cpu().numpy())
else:
fm_recon, z = ae_model.forward(feature_map)
loss, rec_loss = ae_model.calculate_loss(
feature_map, fm_recon, ae_criterion)
latent_norm = torch.norm(z, dim=1)
features.append(
torch.cat([
rec_loss.mean(dim=(1, 2, 3)).unsqueeze(1),
latent_norm.unsqueeze(1)
],
dim=1).detach().cpu().numpy())
# generate and save figures
colors = np.zeros((adv_data.shape[0], 3))
colors[:, 1] = np.linspace(1, 0, adv_data.shape[0])
colors[:, 0] = 1
for j in range(len(features)):
# additionally plot multiple points (or KDE) from training/clean data
f1_i = j * 2
f2_i = f1_i + 1
clean_xs = datasets['train'][:, f1_i][:clean_samples]
clean_ys = datasets['train'][:, f2_i][:clean_samples]
ax[j, l].scatter(clean_xs, clean_ys, color='green')
sns.kdeplot(clean_xs,
clean_ys,
cmap='Greens',
shade=True,
shade_lowest=False,
ax=kde_ax[j, l],
alpha=1.0)
ax[j, l].scatter(features[j][:, 0],
features[j][:, 1],
color=colors)
kde_ax[j, l].scatter(features[j][:, 0],
features[j][:, 1],
color=colors)
ax[j, l].grid(True)
kde_ax[j, l].grid(True)
ax[j, l].set_title(f'{adv_type} on layer {labels[j]}')
kde_ax[j, l].set_title(f'{adv_type} on layer {labels[j]}')
ax[j, l].locator_params(nbins=5)
kde_ax[j, l].locator_params(nbins=5)
print(f'Writing {plot_filename}')
fig.savefig(plot_filename, bbox_inches='tight')
close(fig)
print(f'Writing {kde_plot_filename}')
kde_fig.savefig(kde_plot_filename, bbox_inches='tight')
close(kde_fig)
print(f'FINISHED {str(run_dir)}!')
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
if args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
################################ edits
print("Calculate SVD before getting sample mean and variance")
# svd_result= lib_generation.get_pca(model, args.num_classes, feature_list, train_loader)
# lib_generation.get_pca_incremental(model, args.num_classes, feature_list, train_loader,args)
svd_result = None
print('get sample mean and covariance')
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader, svd_result, args)
################################ edits_end_sample_generator
print('get Mahalanobis scores')
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print('Noise: ' + str(magnitude))
for i in range(num_output):
M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \
True, args.net_type, sample_mean, precision, i, magnitude,svd_result,args)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
print('Out-distribution: ' + out_dist)
for i in range(num_output):
M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \
False, args.net_type, sample_mean, precision, i, magnitude,svd_result,args)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_in)
file_name = os.path.join(
args.outf, 'Mahalanobis_%s_%s_%s.npy' %
(str(magnitude), args.dataset, out_dist))
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
def main():
run_dir = args.run_dir
run_name = run_dir.name
matches = runs_regex.match(run_name)
dataset = matches.group(1)
net_type = matches.group(2)
ae_type = matches.group(3)
args.num_classes = 100 if dataset == 'cifar100' else 10
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + net_type + '_' + dataset + '.pth'
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
device = f'cuda:{args.gpu}'
# load networks
if net_type == 'densenet':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print('load model: ' + net_type)
# load dataset
print('load target data: ', dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
dataset, args.batch_size, in_transform, args.dataroot)
# get the feature maps' sizes
model.eval()
temp_x = torch.rand(1, 3, 32, 32).cuda()
feature_maps = model.feature_list(temp_x)[1]
feature_map_sizes = []
for out in feature_maps:
feature_map_sizes.append(out.size()[1:])
criterion = torch.nn.CrossEntropyLoss()
ae_criterion = torch.nn.MSELoss(reduction='none')
ae_type = ae_type
filters = [128, 128, 128]
latent_size = 64
bn = False
if ae_type == 'vae':
lamb = 1e-7
elif ae_type == 'wae':
lamb = 1e-4
elif ae_type == 'waegan':
if net_type == 'densenet':
lamb = [5e-5, 1e-5, 1e-5, 1e-3]
elif net_type == 'resnet':
lamb = [5e-5, 5e-6, 5e-6, 2e-5, 5e-4]
else:
lamb = None
if ae_type == 'waegan':
ae_models = [
build_ae_for_size(ae_type, sizes, filters, latent_size, lam, bn)
for sizes, lam in zip(feature_map_sizes, lamb)
]
else:
ae_models = [
build_ae_for_size(ae_type, sizes, filters, latent_size, lamb, bn)
for sizes in feature_map_sizes
]
# read in model and autoencoder models
models_filename = f'{run_name}/models.pth'
state_dict = torch.load(models_filename)
for i, _ in enumerate(ae_models):
if isinstance(ae_models[i], torch.nn.Module):
ae_models[i].load_state_dict(state_dict[f'model_{i}'])
else:
for j in range(len(ae_models[i])):
ae_models[i][j].load_state_dict(state_dict[f'model_{i}'][j])
for ae_model in ae_models:
if isinstance(ae_model, torch.nn.Module):
ae_model.eval()
else:
for m in ae_model:
m.eval()
# read in final unsupervised model
classifier_model = 'IF'
run_dir = args.run_dir
name_prefix = 'latent_' if args.latent else ''
name_prefix += 'rec_error_' if args.rec_error else ''
name_prefix += 'both_' if args.both else ''
model_filename = str(run_dir /
f'{name_prefix}final_cv_{classifier_model}.joblib')
assert Path(model_filename).exists()
if_classfier = joblib.load(model_filename)
pgd_iters = [(i + 1) * 10 for i in range(20)]
results_file = run_dir / 'increasing_strength_pgd.pickle'
if results_file.exists():
with open(results_file, 'rb') as results_fd:
results = pickle.load(results_fd)
else:
results = {}
for iters in pgd_iters:
# calculate only if result is not present already
key = f'PGD{iters}'
if key not in results:
# read in adversarial AE encoded samples
dataset_names = [f'clean_{key}', f'adv_{key}', f'noisy_{key}']
datasets = {}
for name in dataset_names:
dataset_path = run_dir / f'ae_encoded_{name}.npy'
if dataset_path.exists():
datasets[name] = np.load(str(dataset_path))
else:
print(f'{dataset_path} is missing!')
test_size = len(datasets[f'clean_{key}'])
X_test = np.concatenate([
datasets[f'clean_{key}'],
datasets[f'adv_{key}'],
datasets[f'noisy_{key}'],
])
y_test = np.concatenate([
np.ones(test_size),
np.zeros(test_size),
np.ones(test_size),
])
# classify with the unsupervised classifier
y_pred = if_classfier.predict(X_test)
try:
y_scores = if_classfier.decision_function(X_test)
except:
y_scores = if_classfier.predict_proba(X_test)[0]
# calculate scores
acc = accuracy_score(y_test, y_pred)
auroc = roc_auc_score(y_test, y_scores)
results[f'acc_{key}'] = acc
results[f'auroc_{key}'] = auroc
# save results
with open(results_file, 'wb') as results_fd:
pickle.dump(results, results_fd)
# gather data to numpy arrays
xs = np.array(pgd_iters)
ys = np.zeros(len(xs))
ys_acc = np.zeros(len(xs))
for i, iters in enumerate(pgd_iters):
key = f'PGD{iters}'
ys[i] = results[f'auroc_{key}']
ys_acc[i] = results[f'acc_{key}']
# generate plots
plot_filename = run_dir / 'increasing_strength_pgd.png'
fig, ax1 = plt.subplots(1, 1, figsize=(16, 9))
color = 'tab:red'
ax1.set_xlabel('PGD iterations')
ax1.set_ylabel('AUROC', color=color)
ax1.plot(xs, ys, color=color)
ax1.tick_params(axis='y', labelcolor=color)
ax2 = ax1.twinx() # instantiate a second axes that shares the same x-axis
color = 'tab:blue'
ax2.set_ylabel('Accuracy',
color=color) # we already handled the x-label with ax1
ax2.plot(xs, ys_acc, color=color)
ax2.tick_params(axis='y', labelcolor=color)
fig.tight_layout() # otherwise the right y-label is slightly clipped
print(f'Writing {plot_filename}')
fig.savefig(plot_filename, bbox_inches='tight')
plot_filename = run_dir / 'auroc_increasing_strength_pgd.png'
fig, ax1 = plt.subplots(1, 1, figsize=(16, 9))
ax1.set_xlabel('PGD iterations')
ax1.set_ylabel('AUROC')
x_ticks = xs[::2]
ax1.set_xticks(x_ticks)
ax1.set_xticklabels([str(x) for x in x_ticks])
ax1.plot(xs, ys, marker='o', linestyle='solid', markeredgecolor='b')
fig.tight_layout() # otherwise the right y-label is slightly clipped
print(f'Writing {plot_filename}')
fig.savefig(plot_filename, bbox_inches='tight')
from sklearn import ensemble
import models
MODEL_DISPATCHER = {
"randomforest":
ensemble.RandomForestClassifier(n_estimators=300, verbose=2, n_jobs=-1),
"extratrees":
ensemble.ExtraTreesClassifier(n_estimators=300, n_jobs=-1, verbose=2),
"resnet34":
models.ResNet34(pretrained=True),
"resnet50":
models.ResNet50(pretrained=True),
"seresnext101":
models.SeResNext101(pretrained=True),
"bert-googleqa":
models.BERTBaseUncased(bert_path='bert-base-uncased', n_outputs=30),
"bert-imdb":
models.BERTBaseUncased(bert_path='bert-base-uncased', n_outputs=1)
}
def setup_and_run(args, criterion, device, train_loader, test_loader,
val_loader, logging, results):
global BEST_ACC
print("\n#### Running REF ####")
# architecture
if args.architecture == "MLP":
model = models.MLP(args.input_dim, args.hidden_dim,
args.output_dim).to(device)
elif args.architecture == "LENET300":
model = models.LeNet300(args.input_dim, args.output_dim).to(device)
elif args.architecture == "LENET5":
model = models.LeNet5(args.input_channels, args.im_size,
args.output_dim).to(device)
elif "VGG" in args.architecture:
assert (args.architecture == "VGG11" or args.architecture == "VGG13"
or args.architecture == "VGG16"
or args.architecture == "VGG19")
model = models.VGG(args.architecture, args.input_channels,
args.im_size, args.output_dim).to(device)
elif args.architecture == "RESNET18":
model = models.ResNet18(args.input_channels, args.im_size,
args.output_dim).to(device)
elif args.architecture == "RESNET34":
model = models.ResNet34(args.input_channels, args.im_size,
args.output_dim).to(device)
elif args.architecture == "RESNET50":
model = models.ResNet50(args.input_channels, args.im_size,
args.output_dim).to(device)
elif args.architecture == "RESNET101":
model = models.ResNet101(args.input_channels, args.im_size,
args.output_dim).to(device)
elif args.architecture == "RESNET152":
model = models.ResNet152(args.input_channels, args.im_size,
args.output_dim).to(device)
else:
print('Architecture type "{0}" not recognized, exiting ...'.format(
args.architecture))
exit()
# optimizer
if args.optimizer == "ADAM":
optimizer = optim.Adam(model.parameters(),
lr=args.learning_rate,
weight_decay=args.weight_decay)
elif args.optimizer == "SGD":
optimizer = optim.SGD(
model.parameters(),
lr=args.learning_rate,
momentum=args.momentum,
nesterov=args.nesterov,
weight_decay=args.weight_decay,
)
else:
print('Optimizer type "{0}" not recognized, exiting ...'.format(
args.optimizer))
exit()
# lr-scheduler
if args.lr_decay == "STEP":
scheduler = optim.lr_scheduler.StepLR(optimizer,
step_size=1,
gamma=args.lr_scale)
elif args.lr_decay == "EXP":
scheduler = optim.lr_scheduler.ExponentialLR(optimizer,
gamma=args.lr_scale)
elif args.lr_decay == "MSTEP":
x = args.lr_interval.split(",")
lri = [int(v) for v in x]
scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=lri,
gamma=args.lr_scale)
args.lr_interval = 1 # lr_interval handled in scheduler!
else:
print('LR decay type "{0}" not recognized, exiting ...'.format(
args.lr_decay))
exit()
init_weights(model, xavier=True)
logging.info(model)
num_parameters = sum([l.nelement() for l in model.parameters()])
logging.info("Number of parameters: %d", num_parameters)
start_epoch = -1
iters = 0 # total no of iterations, used to do many things!
# optionally resume from a checkpoint
if args.eval:
logging.info('Loading checkpoint file "{0}" for evaluation'.format(
args.eval))
if not os.path.isfile(args.eval):
print(
'Checkpoint file "{0}" for evaluation not recognized, exiting ...'
.format(args.eval))
exit()
checkpoint = torch.load(args.eval)
model.load_state_dict(checkpoint["state_dict"])
elif args.resume:
checkpoint_file = args.resume
logging.info('Loading checkpoint file "{0}" to resume'.format(
args.resume))
if not os.path.isfile(checkpoint_file):
print('Checkpoint file "{0}" not recognized, exiting ...'.format(
checkpoint_file))
exit()
checkpoint = torch.load(checkpoint_file)
start_epoch = checkpoint["epoch"]
assert args.architecture == checkpoint["architecture"]
model.load_state_dict(checkpoint["state_dict"])
optimizer.load_state_dict(checkpoint["optimizer"])
scheduler.load_state_dict(checkpoint["scheduler"])
BEST_ACC = checkpoint["best_acc1"]
iters = checkpoint["iters"]
logging.debug("best_acc1: {0}, iters: {1}".format(BEST_ACC, iters))
if not args.eval:
logging.info("Training...")
model.train()
st = timer()
for e in range(start_epoch + 1, args.num_epochs):
for i, (data, target) in enumerate(train_loader):
l = train_step(model, device, data, target, optimizer,
criterion)
if i % args.log_interval == 0:
acc1, acc5 = evaluate(args,
model,
device,
val_loader,
training=True)
logging.info(
"Epoch: {0},\t Iter: {1},\t Loss: {loss:.5f},\t Val-Acc1: {acc1:.2f} "
"(Best: {best:.2f}),\t Val-Acc5: {acc5:.2f}".format(
e, i, loss=l, acc1=acc1, best=BEST_ACC, acc5=acc5))
if iters % args.lr_interval == 0:
lr = args.learning_rate
for param_group in optimizer.param_groups:
lr = param_group["lr"]
scheduler.step()
for param_group in optimizer.param_groups:
if lr != param_group["lr"]:
logging.info("lr: {0}".format(
param_group["lr"])) # print if changed
iters += 1
# save checkpoint
acc1, acc5 = evaluate(args,
model,
device,
val_loader,
training=True)
results.add(
epoch=e,
iteration=i,
train_loss=l,
val_acc1=acc1,
best_val_acc1=BEST_ACC,
)
util.save_checkpoint(
{
"epoch": e,
"architecture": args.architecture,
"state_dict": model.state_dict(),
"optimizer": optimizer.state_dict(),
"scheduler": scheduler.state_dict(),
"best_acc1": BEST_ACC,
"iters": iters,
},
is_best=False,
path=args.save_dir,
)
results.save()
et = timer()
logging.info("Elapsed time: {0} seconds".format(et - st))
acc1, acc5 = evaluate(args, model, device, val_loader, training=True)
logging.info(
"End of training, Val-Acc: {acc1:.2f} (Best: {best:.2f}), Val-Acc5: {acc5:.2f}"
.format(acc1=acc1, best=BEST_ACC, acc5=acc5))
# load saved model
saved_model = torch.load(args.save_name)
model.load_state_dict(saved_model["state_dict"])
# end of training
# eval-set
if args.eval_set != "TRAIN" and args.eval_set != "TEST":
print('Evaluation set "{0}" not recognized ...'.format(args.eval_set))
logging.info("Evaluating REF on the {0} set...".format(args.eval_set))
st = timer()
if args.eval_set == "TRAIN":
acc1, acc5 = evaluate(args, model, device, train_loader)
else:
acc1, acc5 = evaluate(args, model, device, test_loader)
et = timer()
logging.info("Accuracy: top-1: {acc1:.2f}, top-5: {acc5:.2f}%".format(
acc1=acc1, acc5=acc5))
logging.info("Elapsed time: {0} seconds".format(et - st))
def main():
# set the path to pre-trained model and output
pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth"
args.outf = args.outf + args.net_type + "_" + args.dataset + "/"
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == "cifar100":
args.num_classes = 100
if args.dataset == "svhn":
out_dist_list = ["cifar10", "imagenet_resize", "lsun_resize"]
else:
out_dist_list = ["svhn", "imagenet_resize", "lsun_resize"]
# load networks
if args.net_type == "densenet":
if args.dataset == "svhn":
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))
)
else:
model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose(
[
transforms.ToTensor(),
transforms.Normalize(
(125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0),
),
]
)
elif args.net_type == "resnet":
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))
)
in_transform = transforms.Compose(
[
transforms.ToTensor(),
transforms.Normalize(
(0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)
),
]
)
model.cuda()
print("load model: " + args.net_type)
# load dataset
print("load target data: ", args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot
)
# measure the performance
M_list = [0, 0.0005, 0.001, 0.0014, 0.002, 0.0024, 0.005, 0.01, 0.05, 0.1, 0.2]
T_list = [1, 10, 100, 1000]
base_line_list = []
ODIN_best_tnr = [0, 0, 0]
ODIN_best_results = [0, 0, 0]
ODIN_best_temperature = [-1, -1, -1]
ODIN_best_magnitude = [-1, -1, -1]
for T in T_list:
for m in M_list:
magnitude = m
temperature = T
lib_generation.get_posterior(
model,
args.net_type,
test_loader,
magnitude,
temperature,
args.outf,
True,
)
out_count = 0
print("Temperature: " + str(temperature) + " / noise: " + str(magnitude))
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot
)
print("Out-distribution: " + out_dist)
lib_generation.get_posterior(
model,
args.net_type,
out_test_loader,
magnitude,
temperature,
args.outf,
False,
)
if temperature == 1 and magnitude == 0:
test_results = callog.metric(args.outf, ["PoT"])
base_line_list.append(test_results)
else:
val_results = callog.metric(args.outf, ["PoV"])
if ODIN_best_tnr[out_count] < val_results["PoV"]["TNR"]:
ODIN_best_tnr[out_count] = val_results["PoV"]["TNR"]
ODIN_best_results[out_count] = callog.metric(args.outf, ["PoT"])
ODIN_best_temperature[out_count] = temperature
ODIN_best_magnitude[out_count] = magnitude
out_count += 1
# print the results
mtypes = ["TNR", "AUROC", "DTACC", "AUIN", "AUOUT"]
print("Baseline method: in_distribution: " + args.dataset + "==========")
count_out = 0
for results in base_line_list:
print("out_distribution: " + out_dist_list[count_out])
for mtype in mtypes:
print(" {mtype:6s}".format(mtype=mtype), end="")
print("\n{val:6.2f}".format(val=100.0 * results["PoT"]["TNR"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUROC"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["DTACC"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUIN"]), end="")
print(" {val:6.2f}\n".format(val=100.0 * results["PoT"]["AUOUT"]), end="")
print("")
count_out += 1
print("ODIN method: in_distribution: " + args.dataset + "==========")
count_out = 0
for results in ODIN_best_results:
print("out_distribution: " + out_dist_list[count_out])
for mtype in mtypes:
print(" {mtype:6s}".format(mtype=mtype), end="")
print("\n{val:6.2f}".format(val=100.0 * results["PoT"]["TNR"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUROC"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["DTACC"]), end="")
print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUIN"]), end="")
print(" {val:6.2f}\n".format(val=100.0 * results["PoT"]["AUOUT"]), end="")
print("temperature: " + str(ODIN_best_temperature[count_out]))
print("magnitude: " + str(ODIN_best_magnitude[count_out]))
print("")
count_out += 1
def __init__(self, args):
self.args = args
self.epoch = None
self.cluster_predictions_transformation = []
# create dataset
if self.args.data_type == 'image':
image_loaders = loaders.ImageLoader(args)
(self.trainset_first_partition_loader_for_train,
self.trainset_second_partition_loader_for_train,
self.trainset_first_partition_loader_for_infer,
self.trainset_second_partition_loader_for_infer,
self.valset_loader, self.normalize) = image_loaders.get_loaders()
self.batch_normalize = loaders.BatchNormalize(
self.normalize.mean, self.normalize.std, inplace=True, device=torch.cuda.current_device())
elif self.args.data_type == 'text':
text_loaders = loaders.TextLoader(args)
(self.trainset_first_partition_loader_for_train,
self.trainset_second_partition_loader_for_train,
self.trainset_first_partition_loader_for_infer,
self.trainset_second_partition_loader_for_infer,
self.valset_loader, self.normalize) = text_loaders.get_loaders()
if self.args.partition == "1":
self.trainset_loader_for_train = self.trainset_first_partition_loader_for_train
elif self.args.partition == "2":
self.trainset_loader_for_train = self.trainset_second_partition_loader_for_train
print("\nDATASET:", args.dataset_full)
# create model
torch.manual_seed(self.args.execution_seed)
torch.cuda.manual_seed(self.args.execution_seed)
print("=> creating model '{}'".format(self.args.model_name))
if self.args.model_name == "densenetbc100":
self.model = models.DenseNet3(
100, int(self.args.number_of_model_classes))
elif self.args.model_name == "resnet32":
self.model = models.ResNet32(
num_c=self.args.number_of_model_classes)
elif self.args.model_name == "resnet34":
self.model = models.ResNet34(
num_c=self.args.number_of_model_classes)
elif self.args.model_name == "resnet110":
self.model = models.ResNet110(
num_c=self.args.number_of_model_classes)
elif self.args.model_name == "resnet50":
self.model = models.ResNet50(
num_c=self.args.number_of_model_classes)
elif self.args.model_name == "resnet18_":
self.model = models.resnet18_(
num_classes=self.args.number_of_model_classes)
elif self.args.model_name == "resnet50_":
self.model = models.resnet18_(
num_classes=self.args.number_of_model_classes)
elif self.args.model_name == "resnet34_":
self.model = models.resnet34_(
num_classes=self.args.number_of_model_classes)
elif self.args.model_name == "resnet101_":
self.model = models.resnet101_(
num_classes=self.args.number_of_model_classes)
elif self.args.model_name == "wideresnet3410":
self.model = models.Wide_ResNet(
depth=34, widen_factor=10, num_classes=self.args.number_of_model_classes)
#elif self.args.model_name == "vgg":
# self.model = models.VGG19(num_classes=self.args.number_of_model_classes)
#elif self.args.model_name == "efficientnetb0_":
# self.model = models.EfficientNet_(
# num_classes=self.args.number_of_model_classes, width_coefficient=1.0, depth_coefficient=1.0, dropout_rate=0.2)
elif self.args.model_name == "efficientnetb0":
self.model = models.EfficientNetB0(
num_classes=self.args.number_of_model_classes)
elif self.args.model_name == "mobilenetv3":
self.model = models.MobileNetV3(
mode='large', classes_num=self.args.number_of_model_classes, input_size=32, width_multiplier=1)
#############################################################################################################################
#############################################################################################################################
elif self.args.model_name == "textrnn":
self.model = models.TextRNN(
#self.args.text_config,len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings)
len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings, self.args.number_of_model_classes)
##############################################################
#elif self.args.model_name == "textcnn":
# self.model = models.TextCNN(
# len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings, self.args.number_of_model_classes)
#elif self.args.model_name == "textcnn1d":
# self.model = models.TextCNN1D(
# len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings, self.args.number_of_model_classes)
#elif self.args.model_name == "attbilstm":
# self.model = models.AttBiLSTM(
# len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings, self.args.number_of_model_classes)
elif self.args.model_name == "rcnn":
self.model = models.RCNN(
#self.args.text_config,len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings)
len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings, self.args.number_of_model_classes)
elif self.args.model_name == "s2satt":
self.model = models.Seq2SeqAttention(
#self.args.text_config,len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings)
len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings, self.args.number_of_model_classes)
#elif self.args.model_name == "fasttext":
# embed_dim = 64
# self.model = models.FastText(
# #self.args.text_config,len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings)
# #len(train_dataset.get_vocab()), embed_dim, len(train_dataset.get_labels()))
# len(text_loaders.train_set.get_vocab()), embed_dim, len(text_loaders.train_set.get_labels()))
self.model.cuda()
torch.manual_seed(self.args.base_seed)
torch.cuda.manual_seed(self.args.base_seed)
# print and save model arch...
#if self.args.exp_type == "cnn_train":
# recent change!!! recent change!!!
print("\nMODEL:", self.model)
with open(os.path.join(self.args.experiment_path, 'model.arch'), 'w') as file:
print(self.model, file=file)
print("\n$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$")
#collect_batch = next(iter(self.trainset_loader_for_train))
#if self.args.data_type == "image":
# input_data = collect_batch[0][0]#.cuda()
#elif self.args.data_type == "text":
# input_data = collect_batch[0].text#.cuda()
#summary(self.model, input_data=input_data[0])
utils.print_num_params(self.model)
print("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$")
# create loss
#self.criterion = losses.GenericLossSecondPart(self.model.classifier).cuda()
self.criterion = nn.CrossEntropyLoss().cuda()
#total_iteractions = self.args.epochs * self.args.iteractions_per_epoch
# create train
#print(len(self.trainset_loader_for_train))
if self.args.optim.startswith("sgd"):
print("\n$$$$$$$$$$$$$$$")
print("OPTIMIZER: SGD")
print("$$$$$$$$$$$$$$$\n")
initial_learning_rate = float(self.args.optim.split("_")[1][1:])
self.args.epochs = int(self.args.optim.split("_")[2][1:])
weight_decay = float(self.args.optim.split("_")[3][1:])
momentum = float(self.args.optim.split("_")[4][1:])
nesterov = True if (self.args.optim.split("_")[5][1:] == "t") else False
#alpha = float(self.args.optim.split("_")[6][1:])
#beta = float(self.args.optim.split("_")[7][1:])
parameters = self.model.parameters()
self.optimizer = torch.optim.SGD(
parameters,
lr=initial_learning_rate,
weight_decay=weight_decay,
momentum=momentum,
nesterov=nesterov)
print("INITIAL LEARNING RATE: ", initial_learning_rate)
print("TOTAL EPOCHS: ", self.args.epochs)
print("WEIGHT DECAY: ", weight_decay)
print("MOMENTUM: ", momentum)
print("NESTEROV: ", nesterov)
#print("ALPHA: ", alpha)
#print("BETA: ", beta)
elif self.args.optim.startswith("sig"):
print("\n$$$$$$$$$$$$$$$")
print("OPTIMIZER: SIG")
print("$$$$$$$$$$$$$$$\n")
initial_learning_rate = float(self.args.optim.split("_")[1][1:])
self.args.epochs = int(self.args.optim.split("_")[2][1:])
weight_decay = float(self.args.optim.split("_")[3][1:])
momentum = float(self.args.optim.split("_")[4][1:])
nesterov = True if (self.args.optim.split("_")[5][1:] == "t") else False
alpha = float(self.args.optim.split("_")[6][1:])
beta = float(self.args.optim.split("_")[7][1:])
parameters = self.model.parameters()
self.optimizer = torch.optim.SGD(
parameters,
lr=initial_learning_rate,
weight_decay=weight_decay,
momentum=momentum,
nesterov=nesterov)
total_iteractions = self.args.epochs * len(self.trainset_loader_for_train)
sig_function = lambda iteraction: 1/(1+math.exp(alpha*(((iteraction+1)/total_iteractions)-beta)))
self.scheduler = torch.optim.lr_scheduler.LambdaLR(self.optimizer, lr_lambda=sig_function, verbose=True)
print("INITIAL LEARNING RATE: ", initial_learning_rate)
print("TOTAL EPOCHS: ", self.args.epochs)
print("WEIGHT DECAY: ", weight_decay)
print("MOMENTUM: ", momentum)
print("NESTEROV: ", nesterov)
print("ALPHA: ", alpha)
print("BETA: ", beta)
elif self.args.optim.startswith("htd"):
print("\n$$$$$$$$$$$$$$$")
print("OPTIMIZER: HTD")
print("$$$$$$$$$$$$$$$\n")
initial_learning_rate = float(self.args.optim.split("_")[1][1:])
self.args.epochs = int(self.args.optim.split("_")[2][1:])
weight_decay = float(self.args.optim.split("_")[3][1:])
momentum = float(self.args.optim.split("_")[4][1:])
nesterov = True if (self.args.optim.split("_")[5][1:] == "t") else False
l = float(self.args.optim.split("_")[6][1:])
u = float(self.args.optim.split("_")[7][1:])
parameters = self.model.parameters()
self.optimizer = torch.optim.SGD(
parameters,
lr=initial_learning_rate,
weight_decay=weight_decay,
momentum=momentum,
nesterov=nesterov)
#htd_function = lambda epoch: (1-math.tanh(l+(u-l)*((epoch+1)/self.args.epochs)))/2
total_iteractions = self.args.epochs * len(self.trainset_loader_for_train)
htd_function = lambda iteraction: (1-math.tanh(l+(u-l)*((iteraction+1)/total_iteractions)))/2
self.scheduler = torch.optim.lr_scheduler.LambdaLR(self.optimizer, lr_lambda=htd_function, verbose=True)
print("INITIAL LEARNING RATE: ", initial_learning_rate)
print("TOTAL EPOCHS: ", self.args.epochs)
print("WEIGHT DECAY: ", weight_decay)
print("MOMENTUM: ", momentum)
print("NESTEROV: ", nesterov)
print("L: ", l)
print("U: ", u)
elif self.args.optim.startswith("cos"):
print("\n$$$$$$$$$$$$$$$")
print("OPTIMIZER: COS")
print("$$$$$$$$$$$$$$$\n")
initial_learning_rate = float(self.args.optim.split("_")[1][1:])
self.args.epochs = int(self.args.optim.split("_")[2][1:])
weight_decay = float(self.args.optim.split("_")[3][1:])
momentum = float(self.args.optim.split("_")[4][1:])
nesterov = True if (self.args.optim.split("_")[5][1:] == "t") else False
#l = float(self.args.optim.split("_")[6][1:])
#u = float(self.args.optim.split("_")[7][1:])
parameters = self.model.parameters()
self.optimizer = torch.optim.SGD(
parameters,
lr=initial_learning_rate,
weight_decay=weight_decay,
momentum=momentum,
nesterov=nesterov)
#cos_function = lambda epoch: (1+math.cos(math.pi*((epoch+1)/self.args.epochs)))/2
total_iteractions = self.args.epochs * len(self.trainset_loader_for_train)
cos_function = lambda iteraction: (1+math.cos(math.pi*((iteraction+1)/total_iteractions)))/2
self.scheduler = torch.optim.lr_scheduler.LambdaLR(self.optimizer, lr_lambda=cos_function, verbose=True)
print("INITIAL LEARNING RATE: ", initial_learning_rate)
print("TOTAL EPOCHS: ", self.args.epochs)
print("WEIGHT DECAY: ", weight_decay)
print("MOMENTUM: ", momentum)
print("NESTEROV: ", nesterov)
#print("L: ", l)
#print("U: ", u)
elif self.args.optim.startswith("adam"):
print("\n$$$$$$$$$$$$$$$")
print("OPTIMIZER: ADAM")
print("$$$$$$$$$$$$$$$\n")
initial_learning_rate = float(self.args.optim.split("_")[1][1:])
self.args.epochs = int(self.args.optim.split("_")[2][1:])
weight_decay = float(self.args.optim.split("_")[3][1:])
amsgrad = True if (self.args.optim.split("_")[4][1:] == "t") else False
beta_first = float(self.args.optim.split("_")[5][2:])
beta_second = float(self.args.optim.split("_")[6][2:])
parameters = self.model.parameters()
self.optimizer = torch.optim.Adam(
parameters,
lr=initial_learning_rate,
amsgrad=amsgrad,
weight_decay=weight_decay,
betas=(beta_first, beta_second))
print("INITIAL LEARNING RATE: ", initial_learning_rate)
print("TOTAL EPOCHS: ", self.args.epochs)
print("WEIGHT DECAY: ", weight_decay)
print("AMSGRAD: ", amsgrad)
print("BETA_FIRST: ", beta_first)
print("BEST_SECOND: ", beta_second)
elif self.args.optim.startswith("rmsprop"):
print("\n$$$$$$$$$$$$$$$$$$")
print("OPTIMIZER: RMSPROP")
print("$$$$$$$$$$$$$$$$$$\n")
initial_learning_rate = float(self.args.optim.split("_")[1][1:])
self.args.epochs = int(self.args.optim.split("_")[2][1:])
weight_decay = float(self.args.optim.split("_")[3][1:])
momentum = float(self.args.optim.split("_")[4][1:])
centered = True if (self.args.optim.split("_")[5][1:] == "t") else False
alpha = float(self.args.optim.split("_")[6][1:])
parameters = self.model.parameters()
self.optimizer = torch.optim.RMSprop(
parameters,
lr=initial_learning_rate,
weight_decay=weight_decay,
momentum=momentum,
centered=centered,
alpha=alpha)
print("INITIAL LEARNING RATE: ", initial_learning_rate)
print("TOTAL EPOCHS: ", self.args.epochs)
print("WEIGHT DECAY: ", weight_decay)
print("MOMENTUM: ", momentum)
print("ALPHA: ", alpha)
elif self.args.optim.startswith("adagrad"):
print("\n$$$$$$$$$$$$$$$$$$")
print("OPTIMIZER: ADAGRAD")
print("$$$$$$$$$$$$$$$$$$\n")
initial_learning_rate = float(self.args.optim.split("_")[1][1:])
self.args.epochs = int(self.args.optim.split("_")[2][1:])
weight_decay = float(self.args.optim.split("_")[3][1:])
lr_decay = float(self.args.optim.split("_")[4][1:])
initial_accumulator_value = float(self.args.optim.split("_")[5][1:])
parameters = self.model.parameters()
self.optimizer = torch.optim.Adagrad(
parameters,
lr=initial_learning_rate,
weight_decay=weight_decay,
lr_decay=lr_decay,
initial_accumulator_value=initial_accumulator_value)
print("INITIAL LEARNING RATE: ", initial_learning_rate)
print("TOTAL EPOCHS: ", self.args.epochs)
print("WEIGHT DECAY: ", weight_decay)
print("LR DECAY: ", lr_decay)
print("INITIAL ACCUMULATOR VALUE: ", initial_accumulator_value)
#self.scheduler = torch.optim.lr_scheduler.MultiStepLR(
# self.optimizer, milestones=self.args.learning_rate_decay_epochs, gamma=args.learning_rate_decay_rate)
print("\nTRAIN:")
#print(self.criterion)
print(self.optimizer)
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
elif args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
elif args.dataset == 'ham10000':
#out_dist_list = ['cifar10', 'imagenet_resize', 'face', 'face_age', 'isic-2017', 'isic-2016']
#out_dist_list = ['cifar10', 'face', 'face_age', 'isic-2017', 'isic-2016']
#out_dist_list = ['cifar10', 'cifar100', 'svhn', 'imagenet_resize', 'lsun_resize', 'face', 'face_age', 'isic-2017', 'isic-2016']
out_dist_list = [
'ham10000-avg-smoothing', 'ham10000-brightness',
'ham10000-contrast', 'ham10000-dilation', 'ham10000-erosion',
'ham10000-med-smoothing', 'ham10000-rotation', 'ham10000-shift'
]
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
elif args.net_type == 'densenet121':
model = DenseNet121(num_classes=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)).state_dict())
in_transform = transforms.Compose([
transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize((0.7630069, 0.5456578, 0.5700767),
(0.14093237, 0.15263236, 0.17000099))
])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print('get sample mean and covariance')
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader)
print('get Mahalanobis scores', num_output)
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print('Noise: ' + str(magnitude))
for i in range(num_output):
print('layer_num', i)
M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \
True, args.net_type, sample_mean, precision, i, magnitude)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
print('Out-distribution: ' + out_dist)
for i in range(num_output):
M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \
False, args.net_type, sample_mean, precision, i, magnitude)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_in)
file_name = os.path.join(
args.outf, 'Mahalanobis_%s_%s_%s.npy' %
(str(magnitude), args.dataset, out_dist))
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
for i, (name, module) in enumerate(model._modules.items()):
module = recursion_change_bn(model)
for m in model.modules():
if 'Conv' in str(type(m)):
setattr(m, 'padding_mode', 'zeros')
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, _ = data_loader.getTargetDataSet(args.dataset,
args.batch_size,
in_transform, args.dataroot)
test_clean_data = torch.load(args.outf + 'clean_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, 'FGSM'))
test_adv_data = torch.load(args.outf + 'adv_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, 'FGSM'))
test_noisy_data = torch.load(args.outf + 'noisy_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, 'FGSM'))
test_label = torch.load(args.outf + 'label_%s_%s_%s.pth' %
(args.net_type, args.dataset, 'FGSM'))
# set information about feature extaction
model.eval()
temp_x = torch.rand(2, 3, 32, 32).cuda()
temp_x = Variable(temp_x)
temp_list = model.feature_list(temp_x)[1]
num_output = len(temp_list)
feature_list = np.empty(num_output)
count = 0
for out in temp_list:
feature_list[count] = out.size(1)
count += 1
print('get sample mean and covariance')
sample_mean, precision = lib_generation.sample_estimator(
model, args.num_classes, feature_list, train_loader)
print('get Mahalanobis scores')
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print('\nNoise: ' + str(magnitude))
for i in range(num_output):
M_in \
= lib_generation.get_Mahalanobis_score_adv(model, test_clean_data, test_label, \
args.num_classes, args.outf, args.net_type, \
sample_mean, precision, i, magnitude)
M_in = np.asarray(M_in, dtype=np.float32)
if i == 0:
Mahalanobis_in = M_in.reshape((M_in.shape[0], -1))
else:
Mahalanobis_in = np.concatenate(
(Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))),
axis=1)
for i in range(num_output):
M_out \
= lib_generation.get_Mahalanobis_score_adv(model, test_adv_data, test_label, \
args.num_classes, args.outf, args.net_type, \
sample_mean, precision, i, magnitude)
M_out = np.asarray(M_out, dtype=np.float32)
if i == 0:
Mahalanobis_out = M_out.reshape((M_out.shape[0], -1))
else:
Mahalanobis_out = np.concatenate(
(Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))),
axis=1)
for i in range(num_output):
M_noisy \
= lib_generation.get_Mahalanobis_score_adv(model, test_noisy_data, test_label, \
args.num_classes, args.outf, args.net_type, \
sample_mean, precision, i, magnitude)
M_noisy = np.asarray(M_noisy, dtype=np.float32)
if i == 0:
Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1))
else:
Mahalanobis_noisy = np.concatenate(
(Mahalanobis_noisy, M_noisy.reshape(
(M_noisy.shape[0], -1))),
axis=1)
Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32)
Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32)
Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32)
Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy))
Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels(
Mahalanobis_out, Mahalanobis_pos)
file_name = os.path.join(
args.outf, 'Mahalanobis_%s_%s_%s.npy' %
(str(magnitude), args.dataset, 'FGSM'))
Mahalanobis_data = np.concatenate(
(Mahalanobis_data, Mahalanobis_labels), axis=1)
np.save(file_name, Mahalanobis_data)
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
# load networks
if args.net_type == 'densenet':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
test_clean_data, test_adv_data, test_noisy_data, test_label = {}, {}, {}, {}
clean_loaders, adv_loaders, noisy_loaders = {}, {}, {}
adv_types = ['FGSM', 'BIM', 'DeepFool', 'CWL2', 'PGD100'
] if args.adv_type is None else [args.adv_type]
for adv_type in adv_types:
test_clean_data[adv_type] = torch.load(
args.outf + 'clean_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
test_adv_data[adv_type] = torch.load(
args.outf + 'adv_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
test_noisy_data[adv_type] = torch.load(
args.outf + 'noisy_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
test_label[adv_type] = torch.load(
args.outf + 'label_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
clean_loaders[adv_type] = torch.utils.data.DataLoader(
test_clean_data[adv_type], batch_size=args.batch_size)
adv_loaders[adv_type] = torch.utils.data.DataLoader(
test_adv_data[adv_type], batch_size=args.batch_size)
noisy_loaders[adv_type] = torch.utils.data.DataLoader(
test_noisy_data[adv_type], batch_size=args.batch_size)
# get the feature maps' sizes
model.eval()
temp_x = torch.rand(1, 3, 32, 32).cuda()
feature_maps = model.feature_list(temp_x)[1]
feature_map_sizes = []
for out in feature_maps:
feature_map_sizes.append(out.size()[1:])
opt_class = torch.optim.Adam
criterion = torch.nn.MSELoss(reduction='none')
ae_type = args.ae_type
filters = [128, 128, 128]
arch_text = str(filters).replace(', ', '_').replace('(',
'').replace(')', '')
latent_size = 64
bn = False
if ae_type == 'vae':
lamb = 1e-7
elif ae_type == 'wae':
lamb = 1e-4
elif ae_type == 'waegan':
if args.net_type == 'densenet':
lamb = [5e-5, 1e-5, 1e-5, 1e-3]
elif args.net_type == 'resnet':
lamb = [5e-5, 5e-6, 5e-6, 2e-5, 5e-4]
else:
lamb = None
lr = 1e-3
for run_n in range(args.runs):
if ae_type == 'waegan':
ae_models = [
build_ae_for_size(ae_type,
sizes,
filters,
latent_size,
lam,
bn,
info=True)
for sizes, lam in zip(feature_map_sizes, lamb)
]
ae_opts = []
for ae_model in ae_models:
ae_opts.append(
tuple([
opt_class(ae_model[i].parameters(), lr)
for i in range(len(ae_model))
]))
else:
ae_models = [
build_ae_for_size(ae_type,
sizes,
filters,
latent_size,
lamb,
bn,
info=True) for sizes in feature_map_sizes
]
ae_opts = [
opt_class(ae_model.parameters(), lr) for ae_model in ae_models
]
run_name = (f'{args.dataset}_{args.net_type}_deep_{ae_type}_arch_{arch_text}_bn_{bn}' \
f'_latent_{latent_size}_lamb_{lamb}_lr_{lr}_bs_{args.batch_size}_epochs_{args.epochs}_{run_n}') \
.replace('.', '_').replace(',', '_')
print(f'Run name: {run_name}')
writer = SummaryWriter(run_name)
for ae_model in ae_models:
if isinstance(ae_model, torch.nn.Module):
ae_model.train()
else:
for m in ae_model:
m.train()
models_filename = f'{run_name}/models.pth'
models_file = Path(models_filename)
if not models_file.exists():
current_x = 0
with tqdm(initial=current_x, unit_scale=True,
dynamic_ncols=True) as pbar:
for epoch in range(args.epochs):
for ae_model, ae_opt in zip(ae_models, ae_opts):
if isinstance(ae_model, tuple):
encoder_o, decoder_o, discriminator_o = ae_opt
custom_schedule(encoder_o, epoch)
custom_schedule(decoder_o, epoch)
custom_schedule(discriminator_o, epoch)
elif isinstance(ae_model, ConvWAE):
custom_schedule(ae_opt, epoch)
elif isinstance(ae_model, ConvVAE):
custom_schedule(ae_opt, epoch)
for X, _ in train_loader:
X = X.cuda()
y_pred, feature_list = model.feature_list(X)
for i, (feature_map, ae_model, ae_opt) in enumerate(
zip(feature_list, ae_models, ae_opts)):
feature_map = feature_map.detach()
if isinstance(ae_model, tuple):
encoder, decoder, discriminator = ae_model
encoder_o, decoder_o, discriminator_o = ae_opt
z = encoder(feature_map)
fm_recon = decoder(z)
# train discriminator
ones = torch.ones((feature_map.size(0), 1),
device='cuda')
zeros = torch.zeros((feature_map.size(0), 1),
device='cuda')
z_sigma = encoder.z_var**0.5
z_fake = torch.empty(
(X.size(0), discriminator.z_dim),
device='cuda').normal_(std=z_sigma)
dis_fake = discriminator(z_fake)
dis_real = discriminator(z)
fake_loss = F.binary_cross_entropy_with_logits(
dis_fake, ones)
real_loss = F.binary_cross_entropy_with_logits(
dis_real, zeros)
d_loss = discriminator.lamb * (fake_loss +
real_loss)
discriminator_o.zero_grad()
d_loss.backward(retain_graph=True)
discriminator_o.step()
# train autoencoder
# rec_losses = criterion(fm_recon, feature_map).sum(dim=(1, 2, 3))
rec_losses = criterion(
fm_recon, feature_map).mean(dim=(1, 2, 3))
rec_loss = rec_losses.mean()
ae_d_loss = F.binary_cross_entropy_with_logits(
dis_real, ones)
ae_loss = rec_loss + discriminator.lamb * ae_d_loss
encoder_o.zero_grad()
decoder_o.zero_grad()
ae_loss.backward()
encoder_o.step()
decoder_o.step()
writer.add_scalar(f'AE {i}/AE Loss',
ae_loss.item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/AE Reconstruction loss',
rec_loss.item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/AE discriminator loss',
ae_d_loss.item(),
global_step=current_x)
writer.add_scalar(f'AE {i}/Discriminator loss',
d_loss.item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/Discriminator fake loss',
fake_loss.item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/Discriminator mean real output',
torch.sigmoid(dis_real).mean().item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/Discriminator mean fake output',
torch.sigmoid(dis_fake).mean().item(),
global_step=current_x)
else:
if isinstance(ae_model, ConvVAE):
fm_recon, mu, logvar, z = ae_model.forward(
feature_map)
loss, rec_loss, div_loss = ae_model.calculate_loss(
feature_map, fm_recon, criterion, mu,
logvar)
elif isinstance(ae_model, ConvWAE):
fm_recon, z = ae_model.forward(feature_map)
z_sigma = ae_model.z_var**0.5
z_gen = torch.empty(
(feature_map.size(0),
ae_model.z_dim)).normal_(
std=z_sigma).cuda()
loss, rec_loss, dis_loss = ae_model.calculate_loss(
feature_map, fm_recon, criterion,
z_gen, z)
else:
fm_recon, z = ae_model.forward(feature_map)
loss, rec_loss = ae_model.calculate_loss(
feature_map, fm_recon, criterion)
ae_opt.zero_grad()
loss.backward()
ae_opt.step()
if isinstance(ae_model, ConvVAE):
writer.add_scalar(f'AE {i}/Loss',
loss.item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/Reconstruction loss',
rec_loss.mean().item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/Divergence loss',
div_loss.item(),
global_step=current_x)
elif isinstance(ae_model, ConvWAE):
writer.add_scalar(f'AE {i}/Loss',
loss.item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/Reconstruction loss',
rec_loss.mean().item(),
global_step=current_x)
writer.add_scalar(
f'AE {i}/Discrepancy loss',
dis_loss.item(),
global_step=current_x)
else:
writer.add_scalar(f'AE {i}/Loss',
loss.item(),
global_step=current_x)
current_x += 1
pbar.set_description(f'Epoch {epoch}')
pbar.update()
# save models
state_dict = {}
for i, ae_model in enumerate(ae_models):
if isinstance(ae_model, torch.nn.Module):
state_dict[f'model_{i}'] = ae_model.state_dict()
else:
state_dict[f'model_{i}'] = tuple([
ae_model[i].state_dict() for i in range(len(ae_model))
])
torch.save(state_dict, models_filename)
else:
state_dict = torch.load(models_filename)
for i, _ in enumerate(ae_models):
if isinstance(ae_models[i], torch.nn.Module):
ae_models[i].load_state_dict(state_dict[f'model_{i}'])
else:
for j in range(len(ae_models[i])):
ae_models[i][j].load_state_dict(
state_dict[f'model_{i}'][j])
# push entire datasets through the (trained) classifier and trained autoencoder and save the results
for ae_model in ae_models:
if isinstance(ae_model, torch.nn.Module):
ae_model.eval()
else:
for m in ae_model:
m.eval()
# order must be preserved - alternatively use tuples and one list
loaders = [train_loader]
dataset_names = ['train']
datasets = []
for adv_type in adv_types:
loaders.append(clean_loaders[adv_type])
dataset_names.append(f'clean_{adv_type}')
loaders.append(adv_loaders[adv_type])
dataset_names.append(f'adv_{adv_type}')
loaders.append(noisy_loaders[adv_type])
dataset_names.append(f'noisy_{adv_type}')
with torch.no_grad():
for name, loader in zip(dataset_names, loaders):
filename = f'{run_name}/ae_encoded_{name}.npy'
if not Path(filename).exists():
final_features = []
with tqdm(initial=0, unit_scale=True,
dynamic_ncols=True) as pbar:
for t in loader:
if isinstance(t, torch.Tensor):
X = t
else:
X = t[0]
X = X.cuda()
y_pred, feature_list = model.feature_list(X)
features = []
for i, (feature_map, ae_model) in enumerate(
zip(feature_list, ae_models)):
feature_map = feature_map.detach()
if isinstance(ae_model, ConvVAE):
fm_recon, mu, logvar, z = ae_model.forward(
feature_map)
loss, rec_loss, div_loss = ae_model.calculate_loss(
feature_map, fm_recon, criterion, mu,
logvar)
mu_norm = torch.norm(mu, dim=1)
logvar_norm = torch.norm(logvar, dim=1)
features.append(
torch.cat(
[
rec_loss.mean(
dim=(1, 2,
3)).unsqueeze(1),
mu_norm.unsqueeze(1),
logvar_norm.unsqueeze(1)
],
dim=1).detach().cpu().numpy())
elif isinstance(ae_model, ConvWAE):
fm_recon, z = ae_model.forward(feature_map)
z_sigma = ae_model.z_var**0.5
z_gen = torch.empty(
(feature_map.size(0),
ae_model.z_dim)).normal_(
std=z_sigma).cuda()
loss, rec_loss, dis_loss = ae_model.calculate_loss(
feature_map, fm_recon, criterion,
z_gen, z)
latent_norm = torch.norm(z, dim=1)
features.append(
torch.cat(
[
rec_loss.mean(
dim=(1, 2,
3)).unsqueeze(1),
latent_norm.unsqueeze(1)
],
dim=1).detach().cpu().numpy())
elif isinstance(ae_model, tuple):
z = ae_model[0](feature_map)
fm_recon = ae_model[1](z)
dis_output = ae_model[2](z)
latent_norm = torch.norm(z, dim=1)
rec_loss = criterion(
fm_recon,
feature_map).mean(dim=(1, 2, 3))
features.append(
torch.cat(
[
rec_loss.unsqueeze(1),
latent_norm.unsqueeze(1),
dis_output
],
dim=1).detach().cpu().numpy())
else:
fm_recon, z = ae_model.forward(feature_map)
loss, rec_loss = ae_model.calculate_loss(
feature_map, fm_recon, criterion)
latent_norm = torch.norm(z, dim=1)
features.append(
torch.cat(
[
rec_loss.mean(
dim=(1, 2,
3)).unsqueeze(1),
latent_norm.unsqueeze(1)
],
dim=1).detach().cpu().numpy())
final_features.append(
np.concatenate(features, axis=1))
pbar.update()
autoencoded_set = np.concatenate(final_features, axis=0)
datasets.append(autoencoded_set)
np.save(filename, autoencoded_set)
print(f'Saving {name} encoded dataset to {filename}')
else:
print(f'Found {filename}')
datasets.append(np.load(filename))
# and save the entire latent encoding
filename = f'{run_name}/latent_{name}.npy'
if not Path(filename).exists():
final_features = []
with tqdm(initial=0, unit_scale=True,
dynamic_ncols=True) as pbar:
for t in loader:
if isinstance(t, torch.Tensor):
X = t
else:
X = t[0]
X = X.cuda()
y_pred, feature_list = model.feature_list(X)
features = []
for i, (feature_map, ae_model) in enumerate(
zip(feature_list, ae_models)):
feature_map = feature_map.detach()
if isinstance(ae_model, ConvVAE):
fm_recon, mu, logvar, z = ae_model.forward(
feature_map)
elif isinstance(ae_model, ConvWAE):
fm_recon, z = ae_model.forward(feature_map)
elif isinstance(ae_model, tuple):
z = ae_model[0](feature_map)
else:
fm_recon, z = ae_model.forward(feature_map)
features.append(z.detach().cpu().numpy())
final_features.append(
np.concatenate(features, axis=1))
pbar.update()
autoencoded_set = np.concatenate(final_features, axis=0)
datasets.append(autoencoded_set)
np.save(filename, autoencoded_set)
print(
f'Saving {name} latent encoded dataset to {filename}')
else:
print(f'Found {filename}')
# test the model performance when features are replaced by the autoencoded features
def test_ae_dataset(loader):
model.eval()
for ae_model in ae_models:
if isinstance(ae_model, torch.nn.Module):
ae_model.eval()
else:
for m in ae_model:
m.eval()
with torch.no_grad():
with tqdm(initial=0, unit_scale=True,
dynamic_ncols=True) as pbar:
total = 0
keys = ['original', 'all'
] + [i + 1 for i in range(len(ae_models))]
correct = {k: 0 for k in keys}
masks = {k: [False] * len(ae_models) for k in keys}
masks['all'] = [True] * len(ae_models)
for i in range(len(ae_models)):
masks[i + 1][i] = True
for X, y in loader:
X = X.cuda()
y = y.cuda()
total += y.size(0)
for k in keys:
y_pred = model.ae_replaced_forward(
X, ae_models, masks[k])
y_pred_max = y_pred.argmax(dim=1)
correct[k] += (y_pred_max == y).sum().item()
pbar.update()
accs = {k: correct[k] / total for k in keys}
return accs
train_accs = test_ae_dataset(train_loader)
print(f'Trainset accuracy:')
for k, v in train_accs.items():
print(f'{k}: {v}')
test_accs = test_ae_dataset(test_loader)
print(f'Testset accuracy:')
for k, v in test_accs.items():
print(f'{k}: {v}')
print('load noisy labels')
args.label_root = args.label_root + args.dataset + '/' + args.noise_type + '/' + str(
args.noise_fraction)
if os.path.isdir(args.label_root) == False:
print('Err: generate noisy labels first')
else:
args.label_root = args.label_root + '/train_labels.npy'
new_label = torch.load(args.label_root)
train_loader.dataset.train_labels = new_label
print('Model: ', args.net_type)
if args.net_type == 'densenet':
model = models.DenseNet3(100, int(args.numclass))
elif args.net_type == 'resnet34':
model = models.ResNet34(num_c=args.numclass)
if args.cuda:
model.cuda()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.wd)
decreasing_lr = list(map(int, args.decreasing_lr.split(',')))
def train(epoch):
model.train()
total = 0
def main():
if args.validation_src == 'FGSM':
if args.dataset == 'svhn':
out_dist_list = [
'cifar10', 'imagenet_resize', 'lsun_resize', 'FGSM'
]
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize', 'FGSM']
else:
if args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize']
else:
out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize']
outf_load = os.path.join(args.outf,
args.net_type + '_' + args.dataset + 'RealNVP')
outf = os.path.join(
args.outf, args.net_type + '_' + args.dataset + 'RealNVP_magnitude')
if os.path.isdir(outf) == False:
os.mkdir(outf)
# torch.cuda.manual_seed(0)
torch.cuda.set_device(args.cuda_index)
if args.dataset == 'cifar100':
args.num_classes = 100
else:
args.num_classes = 10
with open(
'feature_lists/feature_lists_{}_imagenet_resize_{}_Wlinear.pickle'.
format(args.net_type, args.dataset), 'rb') as f:
[
sample_class_mean, list_features, list_features_test,
list_features_out, A, A_inv, log_abs_det_A_inv
] = pickle.load(f)
pre_trained_net = args.net_type + '_' + args.dataset + '.pth'
pre_trained_net = os.path.join('pre_trained', pre_trained_net)
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.cuda_index)))
else:
model = torch.load(pre_trained_net, map_location="cpu")
for i, (name, module) in enumerate(model._modules.items()):
module = recursion_change_bn(model)
for m in model.modules():
if 'Conv' in str(type(m)):
setattr(m, 'padding_mode', 'zeros')
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
else:
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.cuda_index)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
print('load model: ' + args.net_type)
model.to(device)
model.eval()
# load dataset
print('load in-data: ', args.dataset)
num_layers = len(sample_class_mean)
for layer in range(num_layers):
num_features = A_inv[layer].shape[0]
half_features = int(num_features / 2)
zeros = np.zeros(half_features)
ones = np.ones(half_features)
right = np.concatenate((zeros, ones), axis=None)
left = np.concatenate((ones, zeros), axis=None)
masks = torch.from_numpy(
np.array([
right, left, right, left, right, left, right, left, right, left
]).astype(np.float32)).cuda()
flow = []
# We reduce the number of neurons in the hidden layers due to GPU memory limitations (11 GB in GTX 2080Ti) - comment out this line for larger GPU memory
length_hidden = reture_length_hidden(layer)
A_layer = torch.tensor(A[layer])
A_inv_layer = torch.tensor(A_inv[layer])
log_abs_det_A_inv_layer = torch.tensor(log_abs_det_A_inv[layer])
for i in range(args.num_classes):
MODEL_FLOW = os.path.join(
outf_load,
'model_{}_layer_{}_residual_flow_{}length_hidden'.format(
args.dataset, layer, length_hidden), 'flow_{}'.format(i))
flow.append(
RealNVP(masks, num_features, length_hidden, A_layer,
A_inv_layer, log_abs_det_A_inv_layer))
flow[i].load_state_dict(torch.load(MODEL_FLOW,
map_location="cuda:{}".format(
args.cuda_index)),
strict=False)
flow[i].to(device)
flow[i].eval()
sample_class_mean_layer = sample_class_mean[layer]
m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005]
for magnitude in m_list:
print('Noise: ' + str(magnitude))
_, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
score_in = lib_generation.get_resflow_score(
test_loader, model, layer, args.num_classes, args.net_type,
sample_class_mean_layer, flow, magnitude)
for out_dist in out_dist_list:
print('load out-data: ', out_dist)
if out_dist == 'FGSM':
test_loader, out_loader = data_loader.getFGSM(
args.batch_size, args.dataset, args.net_type)
score_in = lib_generation.get_resflow_score_FGSM(
test_loader, model, layer, args.num_classes,
args.net_type, sample_class_mean_layer, flow,
magnitude)
score_out = lib_generation.get_resflow_score_FGSM(
out_loader, model, layer, args.num_classes,
args.net_type, sample_class_mean_layer, flow,
magnitude)
else:
out_loader = data_loader.getNonTargetDataSet(
out_dist, args.batch_size, in_transform, args.dataroot)
score_out = lib_generation.get_resflow_score(
out_loader, model, layer, args.num_classes,
args.net_type, sample_class_mean_layer, flow,
magnitude)
pram = {
'out_dist': out_dist,
'Network_type': args.net_type,
'Layer': layer,
'Batch_size': args.batch_size,
'cuda_index': args.cuda_index,
'length_hidden': length_hidden,
'dropout': False,
'weight_decay': 0,
'init_zeros': True,
'num_flows': int(len(flow[0].t)),
'magnitude': magnitude,
}
with open(
os.path.join(
outf,
'Residual_flow_%s_%s_layer_%s_%smagnitude.txt' %
(args.dataset, out_dist, layer, magnitude)),
'w') as file:
file.write('date: %s\n' % (datetime.datetime.now()))
file.write(json.dumps(pram))
score_in = np.asarray(score_in, dtype=np.float32)
score_out = np.asarray(score_out, dtype=np.float32)
score_data, score_labels = lib_generation.merge_and_generate_labels(
score_out, score_in)
file_name = os.path.join(
outf, 'Residual_flow_%s_%s_layer_%s_%smagnitude' %
(args.dataset, out_dist, layer, magnitude))
score_data = np.concatenate((score_data, score_labels), axis=1)
np.savez(file_name, score_data, pram)
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
if args.dataset == 'svhn':
out_dist_list = ['cifar10', 'imagenet_resize',] #'lsun_resize']
else:
out_dist_list = ['svhn', 'imagenet_resize',] #'lsun_resize']
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(torch.load(pre_trained_net, map_location = "cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net, map_location = "cuda:" + str(args.gpu))
in_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize([0.5071, 0.4867, 0.4408], [0.2675, 0.2565, 0.2761]),])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(torch.load(pre_trained_net, map_location = "cuda:" + str(args.gpu)))
in_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot)
# measure the performance
M_list = [0, 0.0005, 0.001, 0.0014, 0.002, 0.0024, 0.005, 0.01, 0.05, 0.1, 0.2]
T_list = [1, 10, 100, 1000]
base_line_list = []
ODIN_best_tnr = [0, 0, 0]
ODIN_best_results = [0 , 0, 0]
ODIN_best_temperature = [-1, -1, -1]
ODIN_best_magnitude = [-1, -1, -1]
for T in T_list:
for m in M_list:
magnitude = m
temperature = T
lib_generation.get_posterior(model, args.net_type, test_loader, magnitude, temperature, args.outf, True)
out_count = 0
print('Temperature: ' + str(temperature) + ' / noise: ' + str(magnitude))
for out_dist in out_dist_list:
out_test_loader = data_loader.getNonTargetDataSet(out_dist, args.batch_size, in_transform, args.dataroot)
print('Out-distribution: ' + out_dist)
lib_generation.get_posterior(model, args.net_type, out_test_loader, magnitude, temperature, args.outf, False)
if temperature == 1 and magnitude == 0:
test_results = callog.metric(args.outf, ['PoT'])
base_line_list.append(test_results)
else:
val_results = callog.metric(args.outf, ['PoV'])
if ODIN_best_tnr[out_count] < val_results['PoV']['TNR']:
ODIN_best_tnr[out_count] = val_results['PoV']['TNR']
ODIN_best_results[out_count] = callog.metric(args.outf, ['PoT'])
ODIN_best_temperature[out_count] = temperature
ODIN_best_magnitude[out_count] = magnitude
out_count += 1
# print the results
mtypes = ['TNR', 'AUROC', 'DTACC', 'AUIN', 'AUOUT']
print('Baseline method: in_distribution: ' + args.dataset + '==========')
count_out = 0
for results in base_line_list:
print('out_distribution: '+ out_dist_list[count_out])
for mtype in mtypes:
print(' {mtype:6s}'.format(mtype=mtype), end='')
print('\n{val:6.2f}'.format(val=100.*results['PoT']['TNR']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUROC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['DTACC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUIN']), end='')
print(' {val:6.2f}\n'.format(val=100.*results['PoT']['AUOUT']), end='')
print('')
count_out += 1
print('ODIN method: in_distribution: ' + args.dataset + '==========')
count_out = 0
for results in ODIN_best_results:
print('out_distribution: '+ out_dist_list[count_out])
for mtype in mtypes:
print(' {mtype:6s}'.format(mtype=mtype), end='')
print('\n{val:6.2f}'.format(val=100.*results['PoT']['TNR']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUROC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['DTACC']), end='')
print(' {val:6.2f}'.format(val=100.*results['PoT']['AUIN']), end='')
print(' {val:6.2f}\n'.format(val=100.*results['PoT']['AUOUT']), end='')
print('temperature: ' + str(ODIN_best_temperature[count_out]))
print('magnitude: '+ str(ODIN_best_magnitude[count_out]))
print('')
count_out += 1
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
# load networks
if args.net_type == 'densenet':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255),
(63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)),
])
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
train_loader, test_loader = data_loader.getTargetDataSet(
args.dataset, args.batch_size, in_transform, args.dataroot)
test_clean_data, test_adv_data, test_noisy_data, test_label = {}, {}, {}, {}
clean_loaders, adv_loaders, noisy_loaders = {}, {}, {}
adv_types = ['FGSM', 'BIM', 'DeepFool', 'CWL2', 'PGD100']
for adv_type in adv_types:
test_clean_data[adv_type] = torch.load(
args.outf + 'clean_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
test_adv_data[adv_type] = torch.load(
args.outf + 'adv_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
test_noisy_data[adv_type] = torch.load(
args.outf + 'noisy_data_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
test_label[adv_type] = torch.load(
args.outf + 'label_%s_%s_%s.pth' %
(args.net_type, args.dataset, adv_type))
clean_loaders[adv_type] = torch.utils.data.DataLoader(
test_clean_data[adv_type], batch_size=args.batch_size)
adv_loaders[adv_type] = torch.utils.data.DataLoader(
test_adv_data[adv_type], batch_size=args.batch_size)
noisy_loaders[adv_type] = torch.utils.data.DataLoader(
test_noisy_data[adv_type], batch_size=args.batch_size)
# nasty hack
train_clean_x = []
train_clean_y = []
for X, y in tqdm(train_loader,
desc='Aggregating train dataset',
dynamic_ncols=True):
train_clean_x.append(X.cpu().numpy())
train_clean_y.append(X.cpu().numpy())
train_clean_x = np.concatenate(train_clean_x, axis=0)
train_clean_y = np.concatenate(train_clean_y, axis=0)
X, y = train_clean_x, train_clean_y
model.eval()
# datasets = {}
method_name = 'odd_odds'
dir_name = f'{method_name}_{args.net_type}_{args.dataset}'
os.makedirs(dir_name, exist_ok=True)
# "known" variant
# results_filename = f'{dir_name}/results_known.txt'
# with open(results_filename, 'w') as results_file:
# for adv_type in adv_types:
# train_split = 0.1
# train_size = int(train_split * len(test_clean_data[adv_type]))
# test_size = len(test_clean_data[adv_type]) - train_size
# X = np.concatenate([
# test_clean_data[adv_type][:train_size],
# test_adv_data[adv_type][:train_size],
# test_noisy_data[adv_type][:train_size],
# ])
# label_y = np.concatenate([
# test_label[adv_type][:train_size],
# test_label[adv_type][:train_size],
# test_label[adv_type][:train_size],
# ])
# adv_y = np.concatenate([
# np.ones(train_size),
# np.zeros(train_size),
# np.ones(train_size),
# ])
# X_test = np.concatenate([
# test_clean_data[adv_type][train_size:],
# test_adv_data[adv_type][train_size:],
# test_noisy_data[adv_type][train_size:],
# ])
# label_y_test = np.concatenate([
# test_label[adv_type][train_size:],
# test_label[adv_type][train_size:],
# test_label[adv_type][train_size:],
# ])
# adv_y_test = np.concatenate([
# np.ones(test_size),
# np.zeros(test_size),
# np.ones(test_size),
# ])
# "unsupervised" variant
results_filename = f'{dir_name}/results_unsupervised.txt'
with open(results_filename, 'w') as results_file:
for adv_type in adv_types:
test_size = len(test_clean_data[adv_type])
X_test = np.concatenate([
test_clean_data[adv_type],
test_adv_data[adv_type],
test_noisy_data[adv_type],
])
label_y_test = np.concatenate([
test_label[adv_type],
test_label[adv_type],
test_label[adv_type],
])
adv_y_test = np.concatenate([
np.ones(test_size),
np.zeros(test_size),
np.ones(test_size),
])
adv_y_test_switched = np.concatenate([
np.zeros(test_size),
np.ones(test_size),
np.zeros(test_size),
])
# "train"
class_vectors = model.get_class_vectors()
num_classes = class_vectors.size(0)
# parameters taken from tensorflow file:
# noise_eps = 'n18.0,n24.0,n30.0'.split(',')
# noise_eps_detect = 'n30.0'.split(',')
# parameters taken from pytorch file:
noise_eps = 'n0.01,s0.01,u0.01,n0.02,s0.02,u0.02,s0.03,n0.03,u0.03'.split(
',')
noise_eps_detect = 'n0.003,s0.003,u0.003,n0.005,s0.005,u0.005,s0.008,n0.008,u0.008'.split(
',')
# noise_eps_detect = None
clip_min = min(X.min(), X_test.min())
clip_max = max(X.max(), X_test.max())
predictor = collect_statistics(
X,
y, # these are class labels, not adversarial binary labels
latent_and_logits_fn=model.forward_with_latent,
nb_classes=args.num_classes,
weights=model.get_class_vectors(),
targeted=False,
noise_eps=noise_eps,
noise_eps_detect=noise_eps_detect,
num_noise_samples=256,
batch_size=128,
clip_min=clip_min,
clip_max=clip_max,
p_ratio_cutoff=0.999,
clip_alignments=True,
cache_alignments_dir=dir_name,
)
next(predictor)
# test
y_pred_list = []
y_score_list = []
batch_size = 128
batches, remainder = divmod(X_test.shape[0], batch_size)
if remainder > 0:
batches += 1
for i in tqdm(range(batches),
dynamic_ncols=True,
desc=f"evaluating on {adv_type}"):
X_test_batch = X_test[i * batch_size:(i + 1) * batch_size]
corrected_batch, y_pred_batch, decision_batch = predictor.send(
X_test_batch).T
# print(f'corrected_batch.shape: {corrected_batch.shape}')
# print(f'y_pred_batch.shape: {y_pred_batch.shape}')
# print(f'y_pred_batch: {y_pred_batch}')
y_pred_list.append(y_pred_batch)
y_score_list.append(decision_batch)
y_pred = np.concatenate(y_pred_list, axis=0)
y_scores = np.concatenate(y_score_list, axis=0)
acc = accuracy_score(adv_y_test_switched, y_pred)
auroc = roc_auc_score(adv_y_test_switched, y_scores)
print(f'Accuracy on {adv_type}: {acc}', file=results_file)
print(f'AUROC on {adv_type}: {auroc}', file=results_file)
acc = accuracy_score(adv_y_test, y_pred)
auroc = roc_auc_score(adv_y_test, y_scores)
print(f'Accuracy (original labels) on {adv_type}: {acc}',
file=results_file)
print(f'AUROC (original labels) on {adv_type}: {auroc}',
file=results_file)
utils.train(hyper_param_dict, model, device)
del model
# run ResNet18
model = models.ResNet18()
model.to(device)
project_name = 'ResNet18'
hyper_param_dict['project'] = project_name
hyper_param_dict['lr'] = 0.03
hyper_param_dict['batch'] = 256
print(f'\n\n************** start new model : {project_name} ******************')
utils.train(hyper_param_dict, model, device)
del model
# run ResNet34
model = models.ResNet34()
model.to(device)
project_name = 'ResNet34'
hyper_param_dict['project'] = project_name
print(f'\n\n************** start new model : {project_name} ******************')
utils.train(hyper_param_dict, model, device)
del model
# run ResNet50
# model = models.ResNet50()
# model.to(device)
# project_name = 'ResNet50'
# hyper_param_dict['project'] = project_name
# print(f'\n\n************** start new model : {project_name} ******************')
# utils.train(hyper_param_dict, model, device)
# del model
def train(**kwargs):
'''
训练
'''
# 根据命令行配置
opt.parse(kwargs)
vis = Visualizer(opt.env)
# step1:定义网络
model = models.ResNet34()
# if opt.load_model_path:
# model.load(opt.load_model_path)
# if opt.use_gpu: model.cuda()
# 定义数据
train_data = DogCat(opt.train_data_root, train=True)
val_data = DogCat(opt.train_data_root, train=False)
train_dataloader = DataLoader(train_data,
opt.batch_size,
shuffle=True,
num_workers=opt.num_workers)
val_dataloader = DataLoader(val_data,
opt.batch_size,
shuffle=False,
num_workers=opt.num_workers)
# 定义损失函数和优化器
criterion = t.nn.CrossEntropyLoss()
lr = opt.lr
optimizer = t.optim.Adam(model.parameters(),
lr=lr,
weight_decay=opt.weight_decay)
# 计算重要指标
loss_meter = meter.AverageValueMeter()
confusion_matrix = meter.ConfusionMeter(2)
previous_loss = 1e100
# 开始训练
for epoch in range(opt.max_epoch):
loss_meter.reset()
confusion_matrix.reset()
for ii, (data, label) in tqdm(enumerate(train_dataloader)):
# 训练模型
input = Variable(data)
target = Variable(label)
# if opt.use_gpu:
# input = input.cuda()
# target = target.cuda()
optimizer.zero_grad()
score = model(input)
loss = criterion(score, target)
loss.backward()
optimizer.step()
# meters update and visualize
loss_meter.add(loss.data[0])
confusion_matrix.add(score.data, target.data)
if ii % opt.print_freq == opt.print_freq - 1:
vis.plot('loss', loss_meter.value()[0])
# 进入debug模式
if os.path.exists(opt.debug_file):
import ipdb
ipdb.set_trace()
# validate and visualize
val_cm, val_accuracy = val(model, val_dataloader)
vis.plot('val_accuracy', val_accuracy)
vis.log(
"epoch:{epoch},lr:{lr},loss:{loss},train_cm:{train_cm},val_cm:{val_cm}"
.format(epoch=epoch,
loss=loss_meter.value()[0],
val_cm=str(val_cm.value()),
train_cm=str(confusion_matrix.value()),
lr=lr))
# update learning rate
if loss_meter.value()[0] > previous_loss:
lr = lr * opt.lr_decay
# 第二种降低学习率的方法:不会有moment等信息的丢失
for param_group in optimizer.param_groups:
param_group['lr'] = lr
previous_loss = loss_meter.value()[0]
def main():
# set the path to pre-trained model and output
pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth'
args.outf = args.outf + args.net_type + '_' + args.dataset + '/'
if os.path.isdir(args.outf) == False:
os.mkdir(args.outf)
torch.cuda.manual_seed(0)
torch.cuda.set_device(args.gpu)
# check the in-distribution dataset
if args.dataset == 'cifar100':
args.num_classes = 100
if args.adv_type == 'FGSM':
adv_noise = 0.05
elif args.adv_type == 'BIM':
adv_noise = 0.01
elif args.adv_type == 'DeepFool':
if args.net_type == 'resnet':
if args.dataset == 'cifar10':
adv_noise = 0.18
elif args.dataset == 'cifar100':
adv_noise = 0.03
else:
adv_noise = 0.1
else:
if args.dataset == 'cifar10':
adv_noise = 0.6
elif args.dataset == 'cifar100':
adv_noise = 0.1
else:
adv_noise = 0.5
# load networks
if args.net_type == 'densenet':
if args.dataset == 'svhn':
model = models.DenseNet3(100, int(args.num_classes))
model.load_state_dict(
torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu)))
else:
model = torch.load(pre_trained_net,
map_location="cuda:" + str(args.gpu))
in_transform = transforms.Compose([transforms.ToTensor(), \
transforms.Normalize((125.3/255, 123.0/255, 113.9/255), \
(63.0/255, 62.1/255.0, 66.7/255.0)),])
min_pixel = -1.98888885975
max_pixel = 2.12560367584
if args.dataset == 'cifar10':
if args.adv_type == 'FGSM':
random_noise_size = 0.21 / 4
elif args.adv_type == 'BIM':
random_noise_size = 0.21 / 4
elif args.adv_type == 'DeepFool':
random_noise_size = 0.13 * 2 / 10
elif args.adv_type == 'CWL2':
random_noise_size = 0.03 / 2
elif args.dataset == 'cifar100':
if args.adv_type == 'FGSM':
random_noise_size = 0.21 / 8
elif args.adv_type == 'BIM':
random_noise_size = 0.21 / 8
elif args.adv_type == 'DeepFool':
random_noise_size = 0.13 * 2 / 8
elif args.adv_type == 'CWL2':
random_noise_size = 0.06 / 5
else:
if args.adv_type == 'FGSM':
random_noise_size = 0.21 / 4
elif args.adv_type == 'BIM':
random_noise_size = 0.21 / 4
elif args.adv_type == 'DeepFool':
random_noise_size = 0.16 * 2 / 5
elif args.adv_type == 'CWL2':
random_noise_size = 0.07 / 2
elif args.net_type == 'resnet':
model = models.ResNet34(num_c=args.num_classes)
model.load_state_dict(
torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)))
in_transform = transforms.Compose([transforms.ToTensor(), \
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),])
min_pixel = -2.42906570435
max_pixel = 2.75373125076
if args.dataset == 'cifar10':
if args.adv_type == 'FGSM':
random_noise_size = 0.25 / 4
elif args.adv_type == 'BIM':
random_noise_size = 0.13 / 2
elif args.adv_type == 'DeepFool':
random_noise_size = 0.25 / 4
elif args.adv_type == 'CWL2':
random_noise_size = 0.05 / 2
elif args.dataset == 'cifar100':
if args.adv_type == 'FGSM':
random_noise_size = 0.25 / 8
elif args.adv_type == 'BIM':
random_noise_size = 0.13 / 4
elif args.adv_type == 'DeepFool':
random_noise_size = 0.13 / 4
elif args.adv_type == 'CWL2':
random_noise_size = 0.05 / 2
else:
if args.adv_type == 'FGSM':
random_noise_size = 0.25 / 4
elif args.adv_type == 'BIM':
random_noise_size = 0.13 / 2
elif args.adv_type == 'DeepFool':
random_noise_size = 0.126
elif args.adv_type == 'CWL2':
random_noise_size = 0.05 / 1
model.cuda()
print('load model: ' + args.net_type)
# load dataset
print('load target data: ', args.dataset)
_, test_loader = data_loader.getTargetDataSet(args.dataset,
args.batch_size,
in_transform, args.dataroot)
print('Attack: ' + args.adv_type + ', Dist: ' + args.dataset + '\n')
model.eval()
adv_data_tot, clean_data_tot, noisy_data_tot = 0, 0, 0
label_tot = 0
correct, adv_correct, noise_correct = 0, 0, 0
total, generated_noise = 0, 0
criterion = nn.CrossEntropyLoss().cuda()
selected_list = []
selected_index = 0
for data, target in test_loader:
data, target = data.cuda(), target.cuda()
data, target = Variable(data, volatile=True), Variable(target)
output = model(data)
# compute the accuracy
pred = output.data.max(1)[1]
equal_flag = pred.eq(target.data).cpu()
correct += equal_flag.sum()
noisy_data = torch.add(data.data, random_noise_size,
torch.randn(data.size()).cuda())
noisy_data = torch.clamp(noisy_data, min_pixel, max_pixel)
if total == 0:
clean_data_tot = data.clone().data.cpu()
label_tot = target.clone().data.cpu()
noisy_data_tot = noisy_data.clone().cpu()
else:
clean_data_tot = torch.cat(
(clean_data_tot, data.clone().data.cpu()), 0)
label_tot = torch.cat((label_tot, target.clone().data.cpu()), 0)
noisy_data_tot = torch.cat(
(noisy_data_tot, noisy_data.clone().cpu()), 0)
# generate adversarial
model.zero_grad()
inputs = Variable(data.data, requires_grad=True)
output = model(inputs)
loss = criterion(output, target)
loss.backward()
if args.adv_type == 'FGSM':
gradient = torch.ge(inputs.grad.data, 0)
gradient = (gradient.float() - 0.5) * 2
if args.net_type == 'densenet':
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (63.0/255.0))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (62.1/255.0))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (66.7/255.0))
else:
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (0.2023))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (0.1994))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (0.2010))
elif args.adv_type == 'BIM':
gradient = torch.sign(inputs.grad.data)
for k in range(5):
inputs = torch.add(inputs.data, adv_noise, gradient)
inputs = torch.clamp(inputs, min_pixel, max_pixel)
inputs = Variable(inputs, requires_grad=True)
output = model(inputs)
loss = criterion(output, target)
loss.backward()
gradient = torch.sign(inputs.grad.data)
if args.net_type == 'densenet':
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (63.0/255.0))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (62.1/255.0))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (66.7/255.0))
else:
gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \
gradient.index_select(1, torch.LongTensor([0]).cuda()) / (0.2023))
gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \
gradient.index_select(1, torch.LongTensor([1]).cuda()) / (0.1994))
gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \
gradient.index_select(1, torch.LongTensor([2]).cuda()) / (0.2010))
if args.adv_type == 'DeepFool':
_, adv_data = adversary.deepfool(model, data.data.clone(), target.data.cpu(), \
args.num_classes, step_size=adv_noise, train_mode=False)
adv_data = adv_data.cuda()
elif args.adv_type == 'CWL2':
_, adv_data = adversary.cw(model,
data.data.clone(),
target.data.cpu(),
1.0,
'l2',
crop_frac=1.0)
else:
adv_data = torch.add(inputs.data, adv_noise, gradient)
adv_data = torch.clamp(adv_data, min_pixel, max_pixel)
# measure the noise
temp_noise_max = torch.abs(
(data.data - adv_data).view(adv_data.size(0), -1))
temp_noise_max, _ = torch.max(temp_noise_max, dim=1)
generated_noise += torch.sum(temp_noise_max)
if total == 0:
flag = 1
adv_data_tot = adv_data.clone().cpu()
else:
adv_data_tot = torch.cat((adv_data_tot, adv_data.clone().cpu()), 0)
output = model(Variable(adv_data, volatile=True))
# compute the accuracy
pred = output.data.max(1)[1]
equal_flag_adv = pred.eq(target.data).cpu()
adv_correct += equal_flag_adv.sum()
output = model(Variable(noisy_data, volatile=True))
# compute the accuracy
pred = output.data.max(1)[1]
equal_flag_noise = pred.eq(target.data).cpu()
noise_correct += equal_flag_noise.sum()
for i in range(data.size(0)):
if equal_flag[i] == 1 and equal_flag_noise[
i] == 1 and equal_flag_adv[i] == 0:
selected_list.append(selected_index)
selected_index += 1
total += data.size(0)
selected_list = torch.LongTensor(selected_list)
clean_data_tot = torch.index_select(clean_data_tot, 0, selected_list)
adv_data_tot = torch.index_select(adv_data_tot, 0, selected_list)
noisy_data_tot = torch.index_select(noisy_data_tot, 0, selected_list)
label_tot = torch.index_select(label_tot, 0, selected_list)
torch.save(
clean_data_tot, '%s/clean_data_%s_%s_%s.pth' %
(args.outf, args.net_type, args.dataset, args.adv_type))
torch.save(
adv_data_tot, '%s/adv_data_%s_%s_%s.pth' %
(args.outf, args.net_type, args.dataset, args.adv_type))
torch.save(
noisy_data_tot, '%s/noisy_data_%s_%s_%s.pth' %
(args.outf, args.net_type, args.dataset, args.adv_type))
torch.save(
label_tot, '%s/label_%s_%s_%s.pth' %
(args.outf, args.net_type, args.dataset, args.adv_type))
print('Adversarial Noise:({:.2f})\n'.format(generated_noise / total))
print('Final Accuracy: {}/{} ({:.2f}%)\n'.format(correct, total,
100. * correct / total))
print('Adversarial Accuracy: {}/{} ({:.2f}%)\n'.format(
adv_correct, total, 100. * adv_correct / total))
print('Noisy Accuracy: {}/{} ({:.2f}%)\n'.format(
noise_correct, total, 100. * noise_correct / total))
num_val = 50
if args.dataset == 'cifar100':
num_val = 5
total_train_data, total_train_label, _, _ = data_loader.get_raw_data(
train_loader, num_classes, 0)
total_test_data, total_test_label, val_index, test_index \
= data_loader.get_raw_data(test_loader, num_classes, num_val)
total_val_data, total_val_label, total_test_data, total_test_label \
= data_loader.get_validation(total_test_data, total_test_label, val_index, test_index)
print('load networks: ' + args.net_type)
if args.net_type == 'resnet34':
model = models.ResNet34(num_c=num_classes)
elif args.net_type == 'densenet':
model = models.DenseNet3(100, int(num_classes))
model.load_state_dict(
torch.load(file_root + 'model.pth', map_location="cuda:" + str(args.gpu)))
model.cuda()
print('extact features')
utils.extract_features(model, total_train_data, total_train_label, file_root,
"train_val")
utils.extract_features(model, total_val_data, total_val_label, file_root,
"test_val")
utils.extract_features(model, total_test_data, total_test_label, file_root,
"test_test")
test_data_list, test_label_list = [], []