def defense_predication(self, DefenseModelDirs, defense_name, **kwargs):
# DefenseModelDirs：防御模型所在位置
# defense_name：防御名称（大写）
re_train_defenses = {
'NAT', 'RLT', 'RLT1', 'RLT2', 'RLT3', 'EAT', 'UAPAT', 'NEAT',
'NRC', 'RAT', 'RAT1', 'RAT2', 'RAT3', 'RAT4', 'RAT5', 'RAT6',
'RAT7', 'RAT8', 'RAT9', 'RAT10', 'RAT11', 'MART', 'NEW_MART',
'NEW_MART1', 'NEW_MMA'
}
other_defenses = {'NRC'}
defense_name = defense_name.upper().strip()
assert defense_name in re_train_defenses or input_transformation_defenses or other_defenses
# 如果是重新训练网络防御
if defense_name in re_train_defenses:
print(
'\n##{}## defense is a kind of complete defenses that retrain the model'
.format(defense_name))
# 加载防御模型
defended_model_location = '{}/{}/{}_{}_enhanced.pt'.format(
DefenseModelDirs, defense_name, self.dataset, defense_name)
defended_model = MNIST_CNN().to(
self.device) if self.dataset == 'MNIST' else ResNet18().to(
self.device)
defended_model.load(path=defended_model_location,
device=self.device)
defended_model.eval()
# 进行标签预测
predication = predict(model=defended_model,
samples=self.adv_samples,
device=self.device)
# 返回标签行向量
labels = torch.argmax(predication, 1).cpu().numpy()
return labels
else:
if defense_name == 'NRC':
print(
'\n##{}## defense is a kind of region-based classification defenses ... '
.format(defense_name))
from Defenses.DefenseMethods.NRC import NRCDefense
num_points = 1000
assert 'nrc_radius' in kwargs
assert 'nrc_mean' in kwargs
assert 'nrc_std' in kwargs
radius = kwargs['nrc_radius']
mean = kwargs['nrc_mean']
std = kwargs['nrc_std']
nrc = NRCDefense(model=self.raw_model,
defense_name='NRC',
dataset=self.dataset,
device=self.device,
num_points=num_points)
labels = nrc.region_based_classification(
samples=self.adv_samples,
radius=radius,
mean=mean,
std=std)
return labels
else:
raise ValueError('{} is not supported!!!'.format(defense_name))
class AttackEvaluate:
def __init__(self,
DataSet='MNIST',
AttackName='FGSM',
RawModelLocation='../data/',
CleanDataLocation='../clean_datasets/',
AdvExamplesDir='../AdversarialExampleDatasets/',
device=torch.device('cpu')):
# DataSet：数据集名称
# dataset：数据集名称MNIST或CIFAR10
# AttackName：攻击名称
# attack_name：攻击名称
# RawModelLocation：模型所在位置data/
# CleanDataLocation：干净数据集所在位置clean_datasets/
# AdvExamplesDir：对抗性样本所在位置AdversarialExampleDatasets/
# color_mode：CIFAR10为RGB，MNIST为L
# Targeted：False为非目标攻击，Ture为目标攻击
self.device = device
assert DataSet.upper() in ['MNIST', 'CIFAR10'
], "The data set must be MNIST or CIFAR10"
self.dataset = DataSet.upper()
self.color_mode = 'RGB' if self.dataset == 'CIFAR10' else 'L'
self.attack_name = AttackName.upper()
# 非目标攻击名称
supported_un_targeted = [
'FGSM', 'RFGSM', 'BIM', 'PGD', 'DEEPFOOL', 'UAP'
]
# 目标攻击名称
supported_targeted = ['LLC', "RLLC", 'ILLC', 'JSMA', 'CW2']
assert self.attack_name in supported_un_targeted or self.attack_name in supported_targeted, \
"\nCurrently, our implementation support attacks of FGSM, RFGSM, BIM, UMIFGSM, DeepFool, LLC, RLLC, ILLC, TMIFGSM, JSMA, CW2,....\n"
if self.attack_name.upper() in supported_un_targeted:
self.Targeted = False
else:
self.Targeted = True
# 加载模型
# raw_model_location：模型位置data/CIFAR10/model/CIFAR10_raw.pt或者MNIST
# raw_model：模型
# ********若要衡量白盒攻击将路径改为RawModelLocation/防御名称/数据集名称_防御名称_enhanced.pt********
raw_model_location = '{}{}/model/{}_raw.pt'.format(
RawModelLocation, self.dataset, self.dataset)
if self.dataset == 'MNIST':
self.raw_model = MNIST_CNN().to(device)
self.raw_model.load(path=raw_model_location, device=device)
else:
self.raw_model = ResNet18().to(device)
self.raw_model.load(path=raw_model_location, device=device)
# 获取干净数据集及标签
# nature_samples：干净数据集CleanDatasets/CIFAR10/CIFAR10_inputs.npy或者MNIST
# labels_samples：干净数据集标签CleanDatasets/CIFAR10/CIFAR10_labels.npy或者MNIST
self.nature_samples = np.load('{}{}/{}_inputs.npy'.format(
CleanDataLocation, self.dataset, self.dataset))
self.labels_samples = np.load('{}{}/{}_labels.npy'.format(
CleanDataLocation, self.dataset, self.dataset))
# 获取目标标签
# 如果是LLC RLLC和ILLC攻击，为LLC RLLC和ILLC准备目标标签CleanDatasets/CIFAR10/CIFAR10_llc.npy
if self.attack_name.upper() in ['LLC', 'RLLC', 'ILLC']:
self.targets_samples = np.load('{}{}/{}_llc.npy'.format(
CleanDataLocation, self.dataset, self.dataset))
# 否则目标标签为CleanDatasets/CIFAR10/CIFAR10_targets.npy
else:
self.targets_samples = np.load('{}{}/{}_targets.npy'.format(
CleanDataLocation, self.dataset, self.dataset))
# 获取对抗性样本
# AdvExamplesDir：AdversarialExampleDatasets/attack_name/CIFAR10/或者MNIST
# adv_samples：对抗性样本AdversarialExampleDatasets/attack_name/CIFAR10/attack_name_AdvExamples.npy或者MNIST
self.AdvExamplesDir = AdvExamplesDir + self.attack_name + '/' + self.dataset + '/'
# 如果没有这个路径则提示
if os.path.exists(self.AdvExamplesDir) is False:
print(
"the directory of {} is not existing, please check carefully".
format(self.AdvExamplesDir))
self.adv_samples = np.load('{}{}_AdvExamples.npy'.format(
self.AdvExamplesDir, self.attack_name))
# self.adv_labels = np.load('{}{}_AdvLabels.npy'.format(self.AdvExamplesDir, self.AttackName))
# 对对抗性样本进行标签预测
# predictions：对抗性样本预测标签
predictions = predict(model=self.raw_model,
samples=self.adv_samples,
device=self.device).detach().cpu().numpy()
# 定义softmax函数
def soft_max(x):
return np.exp(x) / np.sum(np.exp(x), axis=0)
# 对预测标签进行softmax计算
# softmax_prediction：经softmax的预测标签
tmp_soft_max = []
for i in range(len(predictions)):
tmp_soft_max.append(soft_max(predictions[i]))
self.softmax_prediction = np.array(tmp_soft_max)
def successful(self, adv_softmax_preds, nature_true_preds, targeted_preds,
target_flag):
# adv_softmax_preds：对抗样本经softmax的预测标签
# nature_true_preds：未经攻击样本的真实标签
# targeted_preds：目标标签
# target_flag：True为目标攻击，False为非目标攻击
# 如果为目标攻击则达到目标标签返回True
if target_flag:
if np.argmax(adv_softmax_preds) == np.argmax(targeted_preds):
return True
else:
return False
# 如果为非目标攻击则与真实标签不同时返回True
else:
if np.argmax(adv_softmax_preds) != np.argmax(nature_true_preds):
return True
else:
return False
# 计算误分类率
# 1 MR:Misclassification Rate
def misclassification_rate(self):
cnt = 0
for i in range(len(self.adv_samples)):
if self.successful(adv_softmax_preds=self.softmax_prediction[i],
nature_true_preds=self.labels_samples[i],
targeted_preds=self.targets_samples[i],
target_flag=self.Targeted):
cnt += 1
mr = cnt / len(self.adv_samples)
print('MR:\t\t{:.1f}%'.format(mr * 100))
return mr
# 4 ASS: Average Structural Similarity
def avg_SSIM(self):
ori_r_channel = np.transpose(np.round(self.nature_samples * 255),
(0, 2, 3, 1)).astype(dtype=np.float32)
adv_r_channel = np.transpose(np.round(self.adv_samples * 255),
(0, 2, 3, 1)).astype(dtype=np.float32)
totalSSIM = 0
cnt = 0
"""
For SSIM function in skimage: http://scikit-image.org/docs/dev/api/skimage.measure.html
multichannel : bool, optional If True, treat the last dimension of the array as channels. Similarity calculations are done
independently for each channel then averaged.
"""
for i in range(len(self.adv_samples)):
if self.successful(adv_softmax_preds=self.softmax_prediction[i],
nature_true_preds=self.labels_samples[i],
targeted_preds=self.targets_samples[i],
target_flag=self.Targeted):
cnt += 1
totalSSIM += SSIM(X=ori_r_channel[i],
Y=adv_r_channel[i],
multichannel=True)
print('ASS:\t{:.3f}'.format(totalSSIM / cnt))
return totalSSIM / cnt
# 6: PSD: Perturbation Sensitivity Distance
def avg_PSD(self):
psd = 0
cnt = 0
for outer in range(len(self.adv_samples)):
if self.successful(
adv_softmax_preds=self.softmax_prediction[outer],
nature_true_preds=self.labels_samples[outer],
targeted_preds=self.targets_samples[outer],
target_flag=self.Targeted):
cnt += 1
image = self.nature_samples[outer]
pert = abs(self.adv_samples[outer] -
self.nature_samples[outer])
for idx_channel in range(image.shape[0]):
image_channel = image[idx_channel]
pert_channel = pert[idx_channel]
image_channel = np.pad(image_channel, 1, 'reflect')
pert_channel = np.pad(pert_channel, 1, 'reflect')
for i in range(1, image_channel.shape[0] - 1):
for j in range(1, image_channel.shape[1] - 1):
psd += pert_channel[i, j] * (1.0 - np.std(
np.array([
image_channel[i - 1, j - 1], image_channel[
i - 1, j], image_channel[i - 1, j + 1],
image_channel[i, j - 1], image_channel[
i, j], image_channel[i, j + 1],
image_channel[i + 1, j - 1], image_channel[
i + 1, j], image_channel[i + 1, j + 1]
])))
print('PSD:\t{:.3f}'.format(psd / cnt))
return psd / cnt
class Generation(object):
__metaclass__ = ABCMeta
def __init__(self,
dataset='MNIST',
attack_name='FGSM',
targeted=False,
raw_model_location='../data/',
clean_data_location='../clean_datasets/',
adv_examples_dir='../AdversarialExampleDatasets/',
device=torch.device('cpu')):
# dataset：数据集MNIST或者CIFAR10
# attack_name：攻击名称
# targeted：目标攻击还非目标攻击
# raw_model_location：训练网络所在位置
# clean_data_location：用于生产对抗样本的干净数据集存放位置
# adv_examples_dir：产生的对抗样本存放位置
# 如果数据集不为MNIST或者CIFAR10，给出提示
self.dataset = dataset.upper()
if self.dataset not in {'MNIST', 'CIFAR10'}:
raise ValueError("The data set must be MNIST or CIFAR10")
# 如果攻击名称不为如下的其中之一，给出提示
self.attack_name = attack_name.upper()
supported = {
'FGSM', 'RFGSM', 'BIM', 'PGD', 'DEEPFOOL', 'LLC', "RLLC", 'ILLC',
'JSMA', 'CW2'
}
if self.attack_name not in supported:
raise ValueError(
self.attack_name +
'is unknown!\nCurrently, our implementation support the attacks: '
+ ', '.join(supported))
# 加载模型和模型参数
# 模型位置data/CIFAR10/model/CIFAR10_raw.pt或者data/MNIST/model/MNIST_raw.pt
# **************根据不同模型可改变模型名称**************
# **************衡量白盒攻击需要改变名称raw_model_location/防御名称/数据集名称_防御名称_enhanced.pt**************
raw_model_location = '{}{}/model/{}_raw.pt'.format(
raw_model_location, self.dataset, self.dataset)
# 根据数据集名称的不同加载不同模型
# **************根据不同模型加载模型参数**************
if self.dataset == 'MNIST':
self.raw_model = MNIST_CNN().to(device)
self.raw_model.load(path=raw_model_location, device=device)
else:
self.raw_model = ResNet18().to(device)
self.raw_model.load(path=raw_model_location, device=device)
# 加载非目标攻击将要被攻击的干净数据集和标签
print(
'Loading the prepared clean samples (nature inputs and corresponding labels) that will be attacked ...... '
)
# 加载将要被攻击的干净数据集clean_datasets/CIFAR10/CIFAR10_inputs.npy或者clean_datasets/MNIST/MNIST_inputs.npy
self.nature_samples = np.load('{}{}/{}_inputs.npy'.format(
clean_data_location, self.dataset, self.dataset))
# 加载将要被攻击的干净标签clean_datasets/CIFAR10/CIFAR10_labels.npy或者clean_datasets/MNIST/MNIST_labels.npy
self.labels_samples = np.load('{}{}/{}_labels.npy'.format(
clean_data_location, self.dataset, self.dataset))
# 如果为目标攻击，获取目标标签
if targeted:
print(
'For Targeted Attacks, loading the randomly selected targeted labels that will be attacked ......'
)
# 如果是LLC, RLLC, ILLC目标攻击，则加载最不可能分类标签clean_datasets/CIFAR10/CIFAR10_llc.npy或者是MNIST
if self.attack_name.upper() in ['LLC', 'RLLC', 'ILLC']:
print(
'#### Especially, for LLC, RLLC, ILLC, loading the least likely class that will be attacked'
)
self.targets_samples = np.load('{}{}/{}_llc.npy'.format(
clean_data_location, self.dataset, self.dataset))
# 否则加载非真实标签的随机标签clean_datasets/CIFAR10/CIFAR10_targets.npy或者是MNIST
else:
self.targets_samples = np.load('{}{}/{}_targets.npy'.format(
clean_data_location, self.dataset, self.dataset))
# 对抗样本的存放位置
# AdversarialExampleDatasets/attact_name/CIFAR10/或者AdversarialExampleDatasets/attact_name/MNIST/
self.adv_examples_dir = adv_examples_dir + self.attack_name + '/' + self.dataset + '/'
# 创建攻击名称文件夹AdversarialExampleDatasets/attact_name
if self.attack_name not in os.listdir(adv_examples_dir):
os.mkdir(adv_examples_dir + self.attack_name + '/')
# 则创建数据集名称文件夹AdversarialExampleDatasets/attact_name/CIFAR10/或者AdversarialExampleDatasets/attact_name/MNIST/
if self.dataset not in os.listdir(adv_examples_dir + self.attack_name +
'/'):
os.mkdir(self.adv_examples_dir)
# 直接创建文件夹
else:
shutil.rmtree('{}'.format(self.adv_examples_dir))
os.mkdir(self.adv_examples_dir)
self.device = device
def generate(self):
print("abstract method of Generation is not implemented")
raise NotImplementedError
def main(args):
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
# dataset：数据集名称MNIST或CIFAR10
dataset = args.dataset.upper()
assert dataset == 'MNIST' or dataset == 'CIFAR10'
# 加载模型和测试集
raw_model_location = '{}{}/model/{}_raw.pt'.format('../data/', dataset,
dataset)
if dataset == 'MNIST':
raw_model = MNIST_CNN().to(device)
raw_model.load(path=raw_model_location, device=device)
test_loader = get_mnist_test_loader(dir_name='../data/MNIST/',
batch_size=30)
else:
raw_model = ResNet18().to(device)
raw_model.load(path=raw_model_location, device=device)
test_loader = get_cifar10_test_loader(dir_name='../data/CIFAR10/',
batch_size=25)
raw_model.eval()
# 原始模型对测试集进行预测
predicted_raw, true_label = prediction(model=raw_model,
test_loader=test_loader,
device=device)
# 需要再训练的防御
re_train_defenses = {'NAT', 'RLT', 'RLT1', 'RLT2', 'RLT3', 'EAT', 'UAPAT'}
# 其他防御
other_defenses = {'NRC'}
# defense_name：防御名称
defense_name = args.defense.upper().strip()
# 如果是再训练模型防御
if defense_name in re_train_defenses:
print(
'\nthe ##{}## defense is a kind of complete defenses that retrain the model'
.format(defense_name))
# 加载防御模型
# defended_model_location：防御模型位置DefenseEnhancedModels/defense_name/CIFAR10_defense_name_enhanced.pt或者MNIST
defended_model_location = '{}/{}/{}_{}_enhanced.pt'.format(
'../DefenseEnhancedModels', defense_name, dataset, defense_name)
defended_model = MNIST_CNN().to(
device) if dataset == 'MNIST' else ResNet18().to(device)
defended_model.load(path=defended_model_location, device=device)
defended_model.eval()
# 利用防御模型进行标签预测
predicted_defended, _ = prediction(model=defended_model,
test_loader=test_loader,
device=device)
# 计算防御指标
raw_acc, def_acc, cav, crr, csr = defense_utility_measure(
predicted_defended, predicted_raw, true_label)
else:
if defense_name == 'NRC':
print(
'\n##{}## defense is a kind of region-based classification defenses ... '
.format(defense_name))
from Defenses.DefenseMethods.NRC import NRCDefense
num_points = 1000
radius = args.radius
mean = args.mean
std = args.std
nrc = NRCDefense(model=raw_model,
defense_name='NRC',
dataset=dataset,
device=device,
num_points=num_points)
predicted_defended = []
with torch.no_grad():
for index, (images, labels) in enumerate(test_loader):
nrc_labels = nrc.region_based_classification(
samples=images, radius=radius, mean=mean, std=std)
predicted_defended.extend(nrc_labels)
predicted_defended = np.array(predicted_defended)
correct_prediction_def = np.equal(predicted_defended, true_label)
def_acc = np.mean(correct_prediction_def.astype(float))
correct_prediction_raw = np.equal(np.argmax(predicted_raw, axis=1),
true_label)
raw_acc = np.mean(correct_prediction_raw.astype(float))
# Classification Accuracy Variance(CAV)
cav = def_acc - raw_acc
# Find the index of correct predicted examples by defence-enhanced model and raw model
idx_def = np.squeeze(np.argwhere(correct_prediction_def == True))
idx_raw = np.squeeze(np.argwhere(correct_prediction_raw == True))
idx = np.intersect1d(idx_def, idx_raw, assume_unique=True)
crr = (len(idx_def) - len(idx)) / len(predicted_raw)
csr = (len(idx_raw) - len(idx)) / len(predicted_raw)
else:
raise ValueError('{} is not supported!!!'.format(defense_name))
# 输出防御指标的值
print("****************************")
print(
"The utility evaluation results of the {} defense for {} Dataset are as follow:"
.format(defense_name, dataset))
print('Acc of Raw Model:\t\t{:.2f}%'.format(raw_acc * 100))
print('Acc of {}-enhanced Model:\t{:.2f}%'.format(defense_name,
def_acc * 100))
print('CAV: {:.2f}%'.format(cav * 100))
print('CRR: {:.2f}%'.format(crr * 100))
print('CSR: {:.2f}%'.format(csr * 100))
print("****************************")
def main(args):
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Set the random seed manually for reproducibility.
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
# dataset：数据集名称MNIST或CIFAR10
# num：选取的干净样本的数量
# dataset_location：数据集所在位置data/CIFAR10/或者MNIST
# raw_model_location：模型所在位置data/CIFAR10/model/CIFAR10_raw.pt或者MNIST
dataset = args.dataset.upper()
num = args.number
# *****************数据集存放的位置*****************
dataset_location = '../data/{}/'.format(dataset)
raw_model_location = '../data/{}/model/{}_raw.pt'.format(dataset, dataset)
print(
"\nStarting to select {} {} Candidates Example, which are correctly classified by the Raw Model from {}\n"
.format(num, dataset, raw_model_location))
# 加载模型，获取测试集
# raw_model：模型
# test_loader：测试集
# load the raw model and testing dataset
assert args.dataset == 'MNIST' or args.dataset == 'CIFAR10'
if dataset == 'MNIST':
raw_model = MNIST_CNN().to(device)
raw_model.load(path=raw_model_location, device=device)
test_loader = get_mnist_test_loader(dir_name=dataset_location,
batch_size=1,
shuffle=False)
else:
raw_model = ResNet18().to(device)
raw_model.load(path=raw_model_location, device=device)
test_loader = get_cifar10_test_loader(dir_name=dataset_location,
batch_size=1,
shuffle=False)
# 获取分类正确的测试集
# successful：测试集经过模型，保留被正确预测的图像和标签以及它们对应softmax最小输出的标签
successful = []
raw_model.eval()
with torch.no_grad():
for image, label in test_loader:
image = image.to(device)
label = label.to(device)
output = raw_model(image)
_, predicted = torch.max(output.data, 1)
if predicted == label:
_, least_likely_class = torch.min(output.data, 1)
successful.append([image, label, least_likely_class])
print(len(successful))
# 随机选取num个正确分类的图像
candidates = random.sample(successful, num)
candidate_images = []
candidate_labels = []
candidates_llc = []
candidate_targets = []
for index in range(len(candidates)):
# 将选择的图片，标签和最不可能的标签分开
image = candidates[index][0].cpu().numpy()
image = np.squeeze(image, axis=0)
candidate_images.append(image)
label = candidates[index][1].cpu().numpy()[0]
llc = candidates[index][2].cpu().numpy()[0]
# 生成0~9的10个标签，去除真实标签，随机选择一个标签
classes = [i for i in range(10)]
classes.remove(label)
target = random.sample(classes, 1)[0]
# 将随机目标标签，最不可能分类目标标签和真实标签转化为one-hot标签保存
one_hot_label = [0 for i in range(10)]
one_hot_label[label] = 1
one_hot_llc = [0 for i in range(10)]
one_hot_llc[llc] = 1
one_hot_target = [0 for i in range(10)]
one_hot_target[target] = 1
candidate_labels.append(one_hot_label)
candidates_llc.append(one_hot_llc)
candidate_targets.append(one_hot_target)
# 图像
candidate_images = np.array(candidate_images)
# 图像对应真实one-hot标签
candidate_labels = np.array(candidate_labels)
# 图像对应最不可能分类的one-hot标签
candidates_llc = np.array(candidates_llc)
# 图像对应非真实标签的随机one-hot标签
candidate_targets = np.array(candidate_targets)
# 打开CIFAR10/或MNIST/文件夹
if dataset not in os.listdir('./'):
os.mkdir('./{}/'.format(dataset))
else:
shutil.rmtree('{}'.format(dataset))
os.mkdir('./{}/'.format(dataset))
# 将图片，标签，最不可能分类的标签，目标标签存入clean_datasets/CIFAR10/CIFAR10_inputs.npy等等或者MNIST
np.save('./{}/{}_inputs.npy'.format(dataset, dataset), candidate_images)
np.save('./{}/{}_labels.npy'.format(dataset, dataset), candidate_labels)
np.save('./{}/{}_llc.npy'.format(dataset, dataset), candidates_llc)
np.save('./{}/{}_targets.npy'.format(dataset, dataset), candidate_targets)
def main(args):
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
# dataset：数据集名称转化为大写形式MNIST/CIFAR10
dataset = args.dataset.upper()
assert dataset == 'MNIST' or dataset == 'CIFAR10'
# batch_size：每个分组的大小为1000
batch_size = 1000
# 获取MNIST/CIFAR10的模型，测试集
model_location = '{}/{}/model/{}_raw.pt'.format('../data', dataset,
dataset)
if dataset == 'MNIST':
raw_model = MNIST_CNN().to(device)
test_loader = get_mnist_test_loader(dir_name='../data/MNIST/',
batch_size=batch_size)
else:
raw_model = ResNet18().to(device)
test_loader = get_cifar10_test_loader(dir_name='../data/CIFAR10/',
batch_size=batch_size)
# 加载MNIST/CIFAR10的模型
raw_model.load(path=model_location, device=device)
# defense_name：防御名称为NRC
defense_name = 'NRC'
# 将参数传入NRC防御中
nrc = NRCDefense(model=raw_model,
defense_name=defense_name,
dataset=dataset,
device=device,
num_points=args.num_points)
# 如果要进行最优半径的搜索
if args.search:
# get the validation dataset (10% with the training dataset)
print('start to search the radius r using validation dataset ...')
# 获取MNIST/CIFAR10的验证集
if dataset == 'MNIST':
_, valid_loader = get_mnist_train_validate_loader(
dir_name='../data/MNIST/',
batch_size=batch_size,
valid_size=0.02,
shuffle=True)
else:
_, valid_loader = get_cifar10_train_validate_loader(
dir_name='../data/CIFAR10/',
batch_size=batch_size,
valid_size=0.02,
shuffle=True)
# radius：通过验证集得到最优的半径值
radius = nrc.search_best_radius(validation_loader=valid_loader,
radius_min=args.radius_min,
radius_max=args.radius_max,
radius_step=args.radius_step)
# 否则半径值为默认的0.01
else:
radius = round(args.radius, 2)
print(
'######\nthe radius for NRC is set or searched as: {}\n######'.format(
radius))
# 计算NRC模型在测试集上的分类精度
print(
'\nStart to calculate the accuracy of region-based classification defense on testing dataset'
)
raw_model.eval()
total = 0.0
correct = 0.0
with torch.no_grad():
for images, labels in test_loader:
nrc_labels = nrc.region_based_classification(samples=images,
radius=radius,
mean=args.mean,
std=args.std)
nrc_labels = torch.from_numpy(nrc_labels)
total += labels.size(0)
correct += (nrc_labels == labels).sum().item()
ratio = correct / total
print(
'\nTest accuracy of the {} model on the testing dataset: {:.1f}/{:.1f} = {:.2f}%\n'
.format(raw_model.model_name, correct, total, ratio * 100))
class SecurityEvaluate:
def __init__(self,
DataSet='MNIST',
AttackName='LLC',
AdvExamplesDir='../AdversarialExampleDatasets/',
device=torch.device('cpu')):
# DataSet：数据集名称
# dataset：数据集名称（大写）
# AttackName：攻击名称
# attack_name：攻击名称（大写）
# AdvExamplesDir：对抗性样本存放位置
self.device = device
assert DataSet.upper() in ['MNIST', 'CIFAR10'
], "The data set must be MNIST or CIFAR10"
self.dataset = DataSet.upper()
# raw_model：加载模型
# ***********不同模型名称***********
raw_model_location = '{}{}/model/{}_raw.pt'.format(
'../data/', self.dataset, self.dataset)
if self.dataset == 'MNIST':
self.raw_model = MNIST_CNN().to(device)
self.raw_model.load(path=raw_model_location, device=device)
else:
self.raw_model = ResNet18().to(device)
self.raw_model.load(path=raw_model_location, device=device)
self.raw_model.eval()
self.attack_name = AttackName.upper()
supported_un_targeted = [
'FGSM', 'RFGSM', 'BIM', 'PGD', 'DEEPFOOL', 'UAP'
]
supported_targeted = ['LLC', "RLLC", 'ILLC', 'JSMA', 'CW2']
assert self.attack_name in supported_un_targeted or self.attack_name in supported_targeted, \
"\nCurrently, our implementation support attacks of FGSM, RFGSM, BIM, UMIFGSM, DeepFool, LLC, RLLC, ILLC, TMIFGSM, JSMA, CW2,....\n"
# 设置Targeted是目标攻击还是非目标攻击
if self.attack_name.upper() in supported_un_targeted:
self.Targeted = False
print('the # {} # attack is a kind of Un-targeted attacks'.format(
self.attack_name))
else:
self.Targeted = True
print('the # {} # attack is a kind of Targeted attacks'.format(
self.attack_name))
# adv_samples，adv_labels，true_labels：加载对抗性样本，对抗性样本标签和真实标签
self.adv_samples = np.load('{}{}/{}/{}_AdvExamples.npy'.format(
AdvExamplesDir, self.attack_name, self.dataset,
self.attack_name)).astype(np.float32)
self.adv_labels = np.load('{}{}/{}/{}_AdvLabels.npy'.format(
AdvExamplesDir, self.attack_name, self.dataset, self.attack_name))
self.true_labels = np.load('{}{}/{}/{}_TrueLabels.npy'.format(
AdvExamplesDir, self.attack_name, self.dataset, self.attack_name))
# targets_samples：获取目标攻击的标签
if self.attack_name.upper() in ['LLC', 'RLLC', 'ILLC']:
self.targets_samples = np.load('{}{}/{}_llc.npy'.format(
'../clean_datasets/', self.dataset, self.dataset))
else:
self.targets_samples = np.load('{}{}/{}_targets.npy'.format(
'../clean_datasets/', self.dataset, self.dataset))
# 对对抗性样本经过防御后的标签预测
def defense_predication(self, DefenseModelDirs, defense_name, **kwargs):
# DefenseModelDirs：防御模型所在位置
# defense_name：防御名称（大写）
re_train_defenses = {
'NAT', 'RLT', 'RLT1', 'RLT2', 'RLT3', 'EAT', 'UAPAT', 'NEAT',
'NRC', 'RAT', 'RAT1', 'RAT2', 'RAT3', 'RAT4', 'RAT5', 'RAT6',
'RAT7', 'RAT8', 'RAT9', 'RAT10', 'RAT11', 'MART', 'NEW_MART',
'NEW_MART1', 'NEW_MMA'
}
other_defenses = {'NRC'}
defense_name = defense_name.upper().strip()
assert defense_name in re_train_defenses or input_transformation_defenses or other_defenses
# 如果是重新训练网络防御
if defense_name in re_train_defenses:
print(
'\n##{}## defense is a kind of complete defenses that retrain the model'
.format(defense_name))
# 加载防御模型
defended_model_location = '{}/{}/{}_{}_enhanced.pt'.format(
DefenseModelDirs, defense_name, self.dataset, defense_name)
defended_model = MNIST_CNN().to(
self.device) if self.dataset == 'MNIST' else ResNet18().to(
self.device)
defended_model.load(path=defended_model_location,
device=self.device)
defended_model.eval()
# 进行标签预测
predication = predict(model=defended_model,
samples=self.adv_samples,
device=self.device)
# 返回标签行向量
labels = torch.argmax(predication, 1).cpu().numpy()
return labels
else:
if defense_name == 'NRC':
print(
'\n##{}## defense is a kind of region-based classification defenses ... '
.format(defense_name))
from Defenses.DefenseMethods.NRC import NRCDefense
num_points = 1000
assert 'nrc_radius' in kwargs
assert 'nrc_mean' in kwargs
assert 'nrc_std' in kwargs
radius = kwargs['nrc_radius']
mean = kwargs['nrc_mean']
std = kwargs['nrc_std']
nrc = NRCDefense(model=self.raw_model,
defense_name='NRC',
dataset=self.dataset,
device=self.device,
num_points=num_points)
labels = nrc.region_based_classification(
samples=self.adv_samples,
radius=radius,
mean=mean,
std=std)
return labels
else:
raise ValueError('{} is not supported!!!'.format(defense_name))
def success_rate(self, defense_predication):
# defense_predication：防御预测结果标签
# adv_labels：对抗性样本标签
# true_labels：真实标签
true_labels = np.argmax(self.true_labels, 1)
# targets：目标攻击的标签
targets = np.argmax(self.targets_samples, 1)
assert defense_predication.shape == true_labels.shape and true_labels.shape == self.adv_labels.shape and self.adv_labels.shape == targets.shape
original_misclassification = 0.0
defense_success = 0.0
for i in range(len(defense_predication)):
# 如果为目标攻击，计算攻击成功条件下，防御成功的数量
if self.Targeted:
if self.adv_labels[i] == targets[i]:
original_misclassification += 1
if defense_predication[i] == true_labels[i]:
defense_success += 1
# 如果为非目标攻击，计算攻击成功条件下，防御成功的数量
else:
if self.adv_labels[i] != true_labels[i]:
original_misclassification += 1
if defense_predication[i] == true_labels[i]:
defense_success += 1
# 返回攻击成功和防御成功的数量
return original_misclassification, defense_success
