def defense_predication(self, DefenseModelDirs, defense_name, **kwargs):
# DefenseModelDirs：防御模型所在位置
# defense_name：防御名称（大写）
re_train_defenses = {
'NAT', 'RLT', 'RLT1', 'RLT2', 'RLT3', 'EAT', 'UAPAT', 'NEAT',
'NRC', 'RAT', 'RAT1', 'RAT2', 'RAT3', 'RAT4', 'RAT5', 'RAT6',
'RAT7', 'RAT8', 'RAT9', 'RAT10', 'RAT11', 'MART', 'NEW_MART',
'NEW_MART1', 'NEW_MMA'
}
other_defenses = {'NRC'}
defense_name = defense_name.upper().strip()
assert defense_name in re_train_defenses or input_transformation_defenses or other_defenses
# 如果是重新训练网络防御
if defense_name in re_train_defenses:
print(
'\n##{}## defense is a kind of complete defenses that retrain the model'
.format(defense_name))
# 加载防御模型
defended_model_location = '{}/{}/{}_{}_enhanced.pt'.format(
DefenseModelDirs, defense_name, self.dataset, defense_name)
defended_model = MNIST_CNN().to(
self.device) if self.dataset == 'MNIST' else ResNet18().to(
self.device)
defended_model.load(path=defended_model_location,
device=self.device)
defended_model.eval()
# 进行标签预测
predication = predict(model=defended_model,
samples=self.adv_samples,
device=self.device)
# 返回标签行向量
labels = torch.argmax(predication, 1).cpu().numpy()
return labels
else:
if defense_name == 'NRC':
print(
'\n##{}## defense is a kind of region-based classification defenses ... '
.format(defense_name))
from Defenses.DefenseMethods.NRC import NRCDefense
num_points = 1000
assert 'nrc_radius' in kwargs
assert 'nrc_mean' in kwargs
assert 'nrc_std' in kwargs
radius = kwargs['nrc_radius']
mean = kwargs['nrc_mean']
std = kwargs['nrc_std']
nrc = NRCDefense(model=self.raw_model,
defense_name='NRC',
dataset=self.dataset,
device=self.device,
num_points=num_points)
labels = nrc.region_based_classification(
samples=self.adv_samples,
radius=radius,
mean=mean,
std=std)
return labels
else:
raise ValueError('{} is not supported!!!'.format(defense_name))
def main(args):
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
# dataset：数据集名称MNIST或CIFAR10
dataset = args.dataset.upper()
assert dataset == 'MNIST' or dataset == 'CIFAR10'
# 加载模型和测试集
raw_model_location = '{}{}/model/{}_raw.pt'.format('../data/', dataset,
dataset)
if dataset == 'MNIST':
raw_model = MNIST_CNN().to(device)
raw_model.load(path=raw_model_location, device=device)
test_loader = get_mnist_test_loader(dir_name='../data/MNIST/',
batch_size=30)
else:
raw_model = ResNet18().to(device)
raw_model.load(path=raw_model_location, device=device)
test_loader = get_cifar10_test_loader(dir_name='../data/CIFAR10/',
batch_size=25)
raw_model.eval()
# 原始模型对测试集进行预测
predicted_raw, true_label = prediction(model=raw_model,
test_loader=test_loader,
device=device)
# 需要再训练的防御
re_train_defenses = {'NAT', 'RLT', 'RLT1', 'RLT2', 'RLT3', 'EAT', 'UAPAT'}
# 其他防御
other_defenses = {'NRC'}
# defense_name：防御名称
defense_name = args.defense.upper().strip()
# 如果是再训练模型防御
if defense_name in re_train_defenses:
print(
'\nthe ##{}## defense is a kind of complete defenses that retrain the model'
.format(defense_name))
# 加载防御模型
# defended_model_location：防御模型位置DefenseEnhancedModels/defense_name/CIFAR10_defense_name_enhanced.pt或者MNIST
defended_model_location = '{}/{}/{}_{}_enhanced.pt'.format(
'../DefenseEnhancedModels', defense_name, dataset, defense_name)
defended_model = MNIST_CNN().to(
device) if dataset == 'MNIST' else ResNet18().to(device)
defended_model.load(path=defended_model_location, device=device)
defended_model.eval()
# 利用防御模型进行标签预测
predicted_defended, _ = prediction(model=defended_model,
test_loader=test_loader,
device=device)
# 计算防御指标
raw_acc, def_acc, cav, crr, csr = defense_utility_measure(
predicted_defended, predicted_raw, true_label)
else:
if defense_name == 'NRC':
print(
'\n##{}## defense is a kind of region-based classification defenses ... '
.format(defense_name))
from Defenses.DefenseMethods.NRC import NRCDefense
num_points = 1000
radius = args.radius
mean = args.mean
std = args.std
nrc = NRCDefense(model=raw_model,
defense_name='NRC',
dataset=dataset,
device=device,
num_points=num_points)
predicted_defended = []
with torch.no_grad():
for index, (images, labels) in enumerate(test_loader):
nrc_labels = nrc.region_based_classification(
samples=images, radius=radius, mean=mean, std=std)
predicted_defended.extend(nrc_labels)
predicted_defended = np.array(predicted_defended)
correct_prediction_def = np.equal(predicted_defended, true_label)
def_acc = np.mean(correct_prediction_def.astype(float))
correct_prediction_raw = np.equal(np.argmax(predicted_raw, axis=1),
true_label)
raw_acc = np.mean(correct_prediction_raw.astype(float))
# Classification Accuracy Variance(CAV)
cav = def_acc - raw_acc
# Find the index of correct predicted examples by defence-enhanced model and raw model
idx_def = np.squeeze(np.argwhere(correct_prediction_def == True))
idx_raw = np.squeeze(np.argwhere(correct_prediction_raw == True))
idx = np.intersect1d(idx_def, idx_raw, assume_unique=True)
crr = (len(idx_def) - len(idx)) / len(predicted_raw)
csr = (len(idx_raw) - len(idx)) / len(predicted_raw)
else:
raise ValueError('{} is not supported!!!'.format(defense_name))
# 输出防御指标的值
print("****************************")
print(
"The utility evaluation results of the {} defense for {} Dataset are as follow:"
.format(defense_name, dataset))
print('Acc of Raw Model:\t\t{:.2f}%'.format(raw_acc * 100))
print('Acc of {}-enhanced Model:\t{:.2f}%'.format(defense_name,
def_acc * 100))
print('CAV: {:.2f}%'.format(cav * 100))
print('CRR: {:.2f}%'.format(crr * 100))
print('CSR: {:.2f}%'.format(csr * 100))
print("****************************")
def main(args):
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Set the random seed manually for reproducibility.
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
# dataset：数据集名称MNIST或CIFAR10
# num：选取的干净样本的数量
# dataset_location：数据集所在位置data/CIFAR10/或者MNIST
# raw_model_location：模型所在位置data/CIFAR10/model/CIFAR10_raw.pt或者MNIST
dataset = args.dataset.upper()
num = args.number
# *****************数据集存放的位置*****************
dataset_location = '../data/{}/'.format(dataset)
raw_model_location = '../data/{}/model/{}_raw.pt'.format(dataset, dataset)
print(
"\nStarting to select {} {} Candidates Example, which are correctly classified by the Raw Model from {}\n"
.format(num, dataset, raw_model_location))
# 加载模型，获取测试集
# raw_model：模型
# test_loader：测试集
# load the raw model and testing dataset
assert args.dataset == 'MNIST' or args.dataset == 'CIFAR10'
if dataset == 'MNIST':
raw_model = MNIST_CNN().to(device)
raw_model.load(path=raw_model_location, device=device)
test_loader = get_mnist_test_loader(dir_name=dataset_location,
batch_size=1,
shuffle=False)
else:
raw_model = ResNet18().to(device)
raw_model.load(path=raw_model_location, device=device)
test_loader = get_cifar10_test_loader(dir_name=dataset_location,
batch_size=1,
shuffle=False)
# 获取分类正确的测试集
# successful：测试集经过模型，保留被正确预测的图像和标签以及它们对应softmax最小输出的标签
successful = []
raw_model.eval()
with torch.no_grad():
for image, label in test_loader:
image = image.to(device)
label = label.to(device)
output = raw_model(image)
_, predicted = torch.max(output.data, 1)
if predicted == label:
_, least_likely_class = torch.min(output.data, 1)
successful.append([image, label, least_likely_class])
print(len(successful))
# 随机选取num个正确分类的图像
candidates = random.sample(successful, num)
candidate_images = []
candidate_labels = []
candidates_llc = []
candidate_targets = []
for index in range(len(candidates)):
# 将选择的图片，标签和最不可能的标签分开
image = candidates[index][0].cpu().numpy()
image = np.squeeze(image, axis=0)
candidate_images.append(image)
label = candidates[index][1].cpu().numpy()[0]
llc = candidates[index][2].cpu().numpy()[0]
# 生成0~9的10个标签，去除真实标签，随机选择一个标签
classes = [i for i in range(10)]
classes.remove(label)
target = random.sample(classes, 1)[0]
# 将随机目标标签，最不可能分类目标标签和真实标签转化为one-hot标签保存
one_hot_label = [0 for i in range(10)]
one_hot_label[label] = 1
one_hot_llc = [0 for i in range(10)]
one_hot_llc[llc] = 1
one_hot_target = [0 for i in range(10)]
one_hot_target[target] = 1
candidate_labels.append(one_hot_label)
candidates_llc.append(one_hot_llc)
candidate_targets.append(one_hot_target)
# 图像
candidate_images = np.array(candidate_images)
# 图像对应真实one-hot标签
candidate_labels = np.array(candidate_labels)
# 图像对应最不可能分类的one-hot标签
candidates_llc = np.array(candidates_llc)
# 图像对应非真实标签的随机one-hot标签
candidate_targets = np.array(candidate_targets)
# 打开CIFAR10/或MNIST/文件夹
if dataset not in os.listdir('./'):
os.mkdir('./{}/'.format(dataset))
else:
shutil.rmtree('{}'.format(dataset))
os.mkdir('./{}/'.format(dataset))
# 将图片，标签，最不可能分类的标签，目标标签存入clean_datasets/CIFAR10/CIFAR10_inputs.npy等等或者MNIST
np.save('./{}/{}_inputs.npy'.format(dataset, dataset), candidate_images)
np.save('./{}/{}_labels.npy'.format(dataset, dataset), candidate_labels)
np.save('./{}/{}_llc.npy'.format(dataset, dataset), candidates_llc)
np.save('./{}/{}_targets.npy'.format(dataset, dataset), candidate_targets)
def main(args):
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
# dataset：数据集名称转化为大写形式MNIST/CIFAR10
dataset = args.dataset.upper()
assert dataset == 'MNIST' or dataset == 'CIFAR10'
# batch_size：每个分组的大小为1000
batch_size = 1000
# 获取MNIST/CIFAR10的模型，测试集
model_location = '{}/{}/model/{}_raw.pt'.format('../data', dataset,
dataset)
if dataset == 'MNIST':
raw_model = MNIST_CNN().to(device)
test_loader = get_mnist_test_loader(dir_name='../data/MNIST/',
batch_size=batch_size)
else:
raw_model = ResNet18().to(device)
test_loader = get_cifar10_test_loader(dir_name='../data/CIFAR10/',
batch_size=batch_size)
# 加载MNIST/CIFAR10的模型
raw_model.load(path=model_location, device=device)
# defense_name：防御名称为NRC
defense_name = 'NRC'
# 将参数传入NRC防御中
nrc = NRCDefense(model=raw_model,
defense_name=defense_name,
dataset=dataset,
device=device,
num_points=args.num_points)
# 如果要进行最优半径的搜索
if args.search:
# get the validation dataset (10% with the training dataset)
print('start to search the radius r using validation dataset ...')
# 获取MNIST/CIFAR10的验证集
if dataset == 'MNIST':
_, valid_loader = get_mnist_train_validate_loader(
dir_name='../data/MNIST/',
batch_size=batch_size,
valid_size=0.02,
shuffle=True)
else:
_, valid_loader = get_cifar10_train_validate_loader(
dir_name='../data/CIFAR10/',
batch_size=batch_size,
valid_size=0.02,
shuffle=True)
# radius：通过验证集得到最优的半径值
radius = nrc.search_best_radius(validation_loader=valid_loader,
radius_min=args.radius_min,
radius_max=args.radius_max,
radius_step=args.radius_step)
# 否则半径值为默认的0.01
else:
radius = round(args.radius, 2)
print(
'######\nthe radius for NRC is set or searched as: {}\n######'.format(
radius))
# 计算NRC模型在测试集上的分类精度
print(
'\nStart to calculate the accuracy of region-based classification defense on testing dataset'
)
raw_model.eval()
total = 0.0
correct = 0.0
with torch.no_grad():
for images, labels in test_loader:
nrc_labels = nrc.region_based_classification(samples=images,
radius=radius,
mean=args.mean,
std=args.std)
nrc_labels = torch.from_numpy(nrc_labels)
total += labels.size(0)
correct += (nrc_labels == labels).sum().item()
ratio = correct / total
print(
'\nTest accuracy of the {} model on the testing dataset: {:.1f}/{:.1f} = {:.2f}%\n'
.format(raw_model.model_name, correct, total, ratio * 100))
class SecurityEvaluate:
def __init__(self,
DataSet='MNIST',
AttackName='LLC',
AdvExamplesDir='../AdversarialExampleDatasets/',
device=torch.device('cpu')):
# DataSet：数据集名称
# dataset：数据集名称（大写）
# AttackName：攻击名称
# attack_name：攻击名称（大写）
# AdvExamplesDir：对抗性样本存放位置
self.device = device
assert DataSet.upper() in ['MNIST', 'CIFAR10'
], "The data set must be MNIST or CIFAR10"
self.dataset = DataSet.upper()
# raw_model：加载模型
# ***********不同模型名称***********
raw_model_location = '{}{}/model/{}_raw.pt'.format(
'../data/', self.dataset, self.dataset)
if self.dataset == 'MNIST':
self.raw_model = MNIST_CNN().to(device)
self.raw_model.load(path=raw_model_location, device=device)
else:
self.raw_model = ResNet18().to(device)
self.raw_model.load(path=raw_model_location, device=device)
self.raw_model.eval()
self.attack_name = AttackName.upper()
supported_un_targeted = [
'FGSM', 'RFGSM', 'BIM', 'PGD', 'DEEPFOOL', 'UAP'
]
supported_targeted = ['LLC', "RLLC", 'ILLC', 'JSMA', 'CW2']
assert self.attack_name in supported_un_targeted or self.attack_name in supported_targeted, \
"\nCurrently, our implementation support attacks of FGSM, RFGSM, BIM, UMIFGSM, DeepFool, LLC, RLLC, ILLC, TMIFGSM, JSMA, CW2,....\n"
# 设置Targeted是目标攻击还是非目标攻击
if self.attack_name.upper() in supported_un_targeted:
self.Targeted = False
print('the # {} # attack is a kind of Un-targeted attacks'.format(
self.attack_name))
else:
self.Targeted = True
print('the # {} # attack is a kind of Targeted attacks'.format(
self.attack_name))
# adv_samples，adv_labels，true_labels：加载对抗性样本，对抗性样本标签和真实标签
self.adv_samples = np.load('{}{}/{}/{}_AdvExamples.npy'.format(
AdvExamplesDir, self.attack_name, self.dataset,
self.attack_name)).astype(np.float32)
self.adv_labels = np.load('{}{}/{}/{}_AdvLabels.npy'.format(
AdvExamplesDir, self.attack_name, self.dataset, self.attack_name))
self.true_labels = np.load('{}{}/{}/{}_TrueLabels.npy'.format(
AdvExamplesDir, self.attack_name, self.dataset, self.attack_name))
# targets_samples：获取目标攻击的标签
if self.attack_name.upper() in ['LLC', 'RLLC', 'ILLC']:
self.targets_samples = np.load('{}{}/{}_llc.npy'.format(
'../clean_datasets/', self.dataset, self.dataset))
else:
self.targets_samples = np.load('{}{}/{}_targets.npy'.format(
'../clean_datasets/', self.dataset, self.dataset))
# 对对抗性样本经过防御后的标签预测
def defense_predication(self, DefenseModelDirs, defense_name, **kwargs):
# DefenseModelDirs：防御模型所在位置
# defense_name：防御名称（大写）
re_train_defenses = {
'NAT', 'RLT', 'RLT1', 'RLT2', 'RLT3', 'EAT', 'UAPAT', 'NEAT',
'NRC', 'RAT', 'RAT1', 'RAT2', 'RAT3', 'RAT4', 'RAT5', 'RAT6',
'RAT7', 'RAT8', 'RAT9', 'RAT10', 'RAT11', 'MART', 'NEW_MART',
'NEW_MART1', 'NEW_MMA'
}
other_defenses = {'NRC'}
defense_name = defense_name.upper().strip()
assert defense_name in re_train_defenses or input_transformation_defenses or other_defenses
# 如果是重新训练网络防御
if defense_name in re_train_defenses:
print(
'\n##{}## defense is a kind of complete defenses that retrain the model'
.format(defense_name))
# 加载防御模型
defended_model_location = '{}/{}/{}_{}_enhanced.pt'.format(
DefenseModelDirs, defense_name, self.dataset, defense_name)
defended_model = MNIST_CNN().to(
self.device) if self.dataset == 'MNIST' else ResNet18().to(
self.device)
defended_model.load(path=defended_model_location,
device=self.device)
defended_model.eval()
# 进行标签预测
predication = predict(model=defended_model,
samples=self.adv_samples,
device=self.device)
# 返回标签行向量
labels = torch.argmax(predication, 1).cpu().numpy()
return labels
else:
if defense_name == 'NRC':
print(
'\n##{}## defense is a kind of region-based classification defenses ... '
.format(defense_name))
from Defenses.DefenseMethods.NRC import NRCDefense
num_points = 1000
assert 'nrc_radius' in kwargs
assert 'nrc_mean' in kwargs
assert 'nrc_std' in kwargs
radius = kwargs['nrc_radius']
mean = kwargs['nrc_mean']
std = kwargs['nrc_std']
nrc = NRCDefense(model=self.raw_model,
defense_name='NRC',
dataset=self.dataset,
device=self.device,
num_points=num_points)
labels = nrc.region_based_classification(
samples=self.adv_samples,
radius=radius,
mean=mean,
std=std)
return labels
else:
raise ValueError('{} is not supported!!!'.format(defense_name))
def success_rate(self, defense_predication):
# defense_predication：防御预测结果标签
# adv_labels：对抗性样本标签
# true_labels：真实标签
true_labels = np.argmax(self.true_labels, 1)
# targets：目标攻击的标签
targets = np.argmax(self.targets_samples, 1)
assert defense_predication.shape == true_labels.shape and true_labels.shape == self.adv_labels.shape and self.adv_labels.shape == targets.shape
original_misclassification = 0.0
defense_success = 0.0
for i in range(len(defense_predication)):
# 如果为目标攻击，计算攻击成功条件下，防御成功的数量
if self.Targeted:
if self.adv_labels[i] == targets[i]:
original_misclassification += 1
if defense_predication[i] == true_labels[i]:
defense_success += 1
# 如果为非目标攻击，计算攻击成功条件下，防御成功的数量
else:
if self.adv_labels[i] != true_labels[i]:
original_misclassification += 1
if defense_predication[i] == true_labels[i]:
defense_success += 1
# 返回攻击成功和防御成功的数量
return original_misclassification, defense_success
