def defense_predication(self, DefenseModelDirs, defense_name, **kwargs): # DefenseModelDirs:防御模型所在位置 # defense_name:防御名称(大写) re_train_defenses = { 'NAT', 'RLT', 'RLT1', 'RLT2', 'RLT3', 'EAT', 'UAPAT', 'NEAT', 'NRC', 'RAT', 'RAT1', 'RAT2', 'RAT3', 'RAT4', 'RAT5', 'RAT6', 'RAT7', 'RAT8', 'RAT9', 'RAT10', 'RAT11', 'MART', 'NEW_MART', 'NEW_MART1', 'NEW_MMA' } other_defenses = {'NRC'} defense_name = defense_name.upper().strip() assert defense_name in re_train_defenses or input_transformation_defenses or other_defenses # 如果是重新训练网络防御 if defense_name in re_train_defenses: print( '\n##{}## defense is a kind of complete defenses that retrain the model' .format(defense_name)) # 加载防御模型 defended_model_location = '{}/{}/{}_{}_enhanced.pt'.format( DefenseModelDirs, defense_name, self.dataset, defense_name) defended_model = MNIST_CNN().to( self.device) if self.dataset == 'MNIST' else ResNet18().to( self.device) defended_model.load(path=defended_model_location, device=self.device) defended_model.eval() # 进行标签预测 predication = predict(model=defended_model, samples=self.adv_samples, device=self.device) # 返回标签行向量 labels = torch.argmax(predication, 1).cpu().numpy() return labels else: if defense_name == 'NRC': print( '\n##{}## defense is a kind of region-based classification defenses ... ' .format(defense_name)) from Defenses.DefenseMethods.NRC import NRCDefense num_points = 1000 assert 'nrc_radius' in kwargs assert 'nrc_mean' in kwargs assert 'nrc_std' in kwargs radius = kwargs['nrc_radius'] mean = kwargs['nrc_mean'] std = kwargs['nrc_std'] nrc = NRCDefense(model=self.raw_model, defense_name='NRC', dataset=self.dataset, device=self.device, num_points=num_points) labels = nrc.region_based_classification( samples=self.adv_samples, radius=radius, mean=mean, std=std) return labels else: raise ValueError('{} is not supported!!!'.format(defense_name))
def main(args): os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.manual_seed(args.seed) if torch.cuda.is_available(): torch.cuda.manual_seed(args.seed) np.random.seed(args.seed) random.seed(args.seed) # dataset:数据集名称MNIST或CIFAR10 dataset = args.dataset.upper() assert dataset == 'MNIST' or dataset == 'CIFAR10' # 加载模型和测试集 raw_model_location = '{}{}/model/{}_raw.pt'.format('../data/', dataset, dataset) if dataset == 'MNIST': raw_model = MNIST_CNN().to(device) raw_model.load(path=raw_model_location, device=device) test_loader = get_mnist_test_loader(dir_name='../data/MNIST/', batch_size=30) else: raw_model = ResNet18().to(device) raw_model.load(path=raw_model_location, device=device) test_loader = get_cifar10_test_loader(dir_name='../data/CIFAR10/', batch_size=25) raw_model.eval() # 原始模型对测试集进行预测 predicted_raw, true_label = prediction(model=raw_model, test_loader=test_loader, device=device) # 需要再训练的防御 re_train_defenses = {'NAT', 'RLT', 'RLT1', 'RLT2', 'RLT3', 'EAT', 'UAPAT'} # 其他防御 other_defenses = {'NRC'} # defense_name:防御名称 defense_name = args.defense.upper().strip() # 如果是再训练模型防御 if defense_name in re_train_defenses: print( '\nthe ##{}## defense is a kind of complete defenses that retrain the model' .format(defense_name)) # 加载防御模型 # defended_model_location:防御模型位置DefenseEnhancedModels/defense_name/CIFAR10_defense_name_enhanced.pt或者MNIST defended_model_location = '{}/{}/{}_{}_enhanced.pt'.format( '../DefenseEnhancedModels', defense_name, dataset, defense_name) defended_model = MNIST_CNN().to( device) if dataset == 'MNIST' else ResNet18().to(device) defended_model.load(path=defended_model_location, device=device) defended_model.eval() # 利用防御模型进行标签预测 predicted_defended, _ = prediction(model=defended_model, test_loader=test_loader, device=device) # 计算防御指标 raw_acc, def_acc, cav, crr, csr = defense_utility_measure( predicted_defended, predicted_raw, true_label) else: if defense_name == 'NRC': print( '\n##{}## defense is a kind of region-based classification defenses ... ' .format(defense_name)) from Defenses.DefenseMethods.NRC import NRCDefense num_points = 1000 radius = args.radius mean = args.mean std = args.std nrc = NRCDefense(model=raw_model, defense_name='NRC', dataset=dataset, device=device, num_points=num_points) predicted_defended = [] with torch.no_grad(): for index, (images, labels) in enumerate(test_loader): nrc_labels = nrc.region_based_classification( samples=images, radius=radius, mean=mean, std=std) predicted_defended.extend(nrc_labels) predicted_defended = np.array(predicted_defended) correct_prediction_def = np.equal(predicted_defended, true_label) def_acc = np.mean(correct_prediction_def.astype(float)) correct_prediction_raw = np.equal(np.argmax(predicted_raw, axis=1), true_label) raw_acc = np.mean(correct_prediction_raw.astype(float)) # Classification Accuracy Variance(CAV) cav = def_acc - raw_acc # Find the index of correct predicted examples by defence-enhanced model and raw model idx_def = np.squeeze(np.argwhere(correct_prediction_def == True)) idx_raw = np.squeeze(np.argwhere(correct_prediction_raw == True)) idx = np.intersect1d(idx_def, idx_raw, assume_unique=True) crr = (len(idx_def) - len(idx)) / len(predicted_raw) csr = (len(idx_raw) - len(idx)) / len(predicted_raw) else: raise ValueError('{} is not supported!!!'.format(defense_name)) # 输出防御指标的值 print("****************************") print( "The utility evaluation results of the {} defense for {} Dataset are as follow:" .format(defense_name, dataset)) print('Acc of Raw Model:\t\t{:.2f}%'.format(raw_acc * 100)) print('Acc of {}-enhanced Model:\t{:.2f}%'.format(defense_name, def_acc * 100)) print('CAV: {:.2f}%'.format(cav * 100)) print('CRR: {:.2f}%'.format(crr * 100)) print('CSR: {:.2f}%'.format(csr * 100)) print("****************************")
def main(args): os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # Set the random seed manually for reproducibility. torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.manual_seed(args.seed) if torch.cuda.is_available(): torch.cuda.manual_seed(args.seed) np.random.seed(args.seed) random.seed(args.seed) # dataset:数据集名称MNIST或CIFAR10 # num:选取的干净样本的数量 # dataset_location:数据集所在位置data/CIFAR10/或者MNIST # raw_model_location:模型所在位置data/CIFAR10/model/CIFAR10_raw.pt或者MNIST dataset = args.dataset.upper() num = args.number # *****************数据集存放的位置***************** dataset_location = '../data/{}/'.format(dataset) raw_model_location = '../data/{}/model/{}_raw.pt'.format(dataset, dataset) print( "\nStarting to select {} {} Candidates Example, which are correctly classified by the Raw Model from {}\n" .format(num, dataset, raw_model_location)) # 加载模型,获取测试集 # raw_model:模型 # test_loader:测试集 # load the raw model and testing dataset assert args.dataset == 'MNIST' or args.dataset == 'CIFAR10' if dataset == 'MNIST': raw_model = MNIST_CNN().to(device) raw_model.load(path=raw_model_location, device=device) test_loader = get_mnist_test_loader(dir_name=dataset_location, batch_size=1, shuffle=False) else: raw_model = ResNet18().to(device) raw_model.load(path=raw_model_location, device=device) test_loader = get_cifar10_test_loader(dir_name=dataset_location, batch_size=1, shuffle=False) # 获取分类正确的测试集 # successful:测试集经过模型,保留被正确预测的图像和标签以及它们对应softmax最小输出的标签 successful = [] raw_model.eval() with torch.no_grad(): for image, label in test_loader: image = image.to(device) label = label.to(device) output = raw_model(image) _, predicted = torch.max(output.data, 1) if predicted == label: _, least_likely_class = torch.min(output.data, 1) successful.append([image, label, least_likely_class]) print(len(successful)) # 随机选取num个正确分类的图像 candidates = random.sample(successful, num) candidate_images = [] candidate_labels = [] candidates_llc = [] candidate_targets = [] for index in range(len(candidates)): # 将选择的图片,标签和最不可能的标签分开 image = candidates[index][0].cpu().numpy() image = np.squeeze(image, axis=0) candidate_images.append(image) label = candidates[index][1].cpu().numpy()[0] llc = candidates[index][2].cpu().numpy()[0] # 生成0~9的10个标签,去除真实标签,随机选择一个标签 classes = [i for i in range(10)] classes.remove(label) target = random.sample(classes, 1)[0] # 将随机目标标签,最不可能分类目标标签和真实标签转化为one-hot标签保存 one_hot_label = [0 for i in range(10)] one_hot_label[label] = 1 one_hot_llc = [0 for i in range(10)] one_hot_llc[llc] = 1 one_hot_target = [0 for i in range(10)] one_hot_target[target] = 1 candidate_labels.append(one_hot_label) candidates_llc.append(one_hot_llc) candidate_targets.append(one_hot_target) # 图像 candidate_images = np.array(candidate_images) # 图像对应真实one-hot标签 candidate_labels = np.array(candidate_labels) # 图像对应最不可能分类的one-hot标签 candidates_llc = np.array(candidates_llc) # 图像对应非真实标签的随机one-hot标签 candidate_targets = np.array(candidate_targets) # 打开CIFAR10/或MNIST/文件夹 if dataset not in os.listdir('./'): os.mkdir('./{}/'.format(dataset)) else: shutil.rmtree('{}'.format(dataset)) os.mkdir('./{}/'.format(dataset)) # 将图片,标签,最不可能分类的标签,目标标签存入clean_datasets/CIFAR10/CIFAR10_inputs.npy等等或者MNIST np.save('./{}/{}_inputs.npy'.format(dataset, dataset), candidate_images) np.save('./{}/{}_labels.npy'.format(dataset, dataset), candidate_labels) np.save('./{}/{}_llc.npy'.format(dataset, dataset), candidates_llc) np.save('./{}/{}_targets.npy'.format(dataset, dataset), candidate_targets)
def main(args): os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_index device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.manual_seed(args.seed) if torch.cuda.is_available(): torch.cuda.manual_seed(args.seed) np.random.seed(args.seed) random.seed(args.seed) # dataset:数据集名称转化为大写形式MNIST/CIFAR10 dataset = args.dataset.upper() assert dataset == 'MNIST' or dataset == 'CIFAR10' # batch_size:每个分组的大小为1000 batch_size = 1000 # 获取MNIST/CIFAR10的模型,测试集 model_location = '{}/{}/model/{}_raw.pt'.format('../data', dataset, dataset) if dataset == 'MNIST': raw_model = MNIST_CNN().to(device) test_loader = get_mnist_test_loader(dir_name='../data/MNIST/', batch_size=batch_size) else: raw_model = ResNet18().to(device) test_loader = get_cifar10_test_loader(dir_name='../data/CIFAR10/', batch_size=batch_size) # 加载MNIST/CIFAR10的模型 raw_model.load(path=model_location, device=device) # defense_name:防御名称为NRC defense_name = 'NRC' # 将参数传入NRC防御中 nrc = NRCDefense(model=raw_model, defense_name=defense_name, dataset=dataset, device=device, num_points=args.num_points) # 如果要进行最优半径的搜索 if args.search: # get the validation dataset (10% with the training dataset) print('start to search the radius r using validation dataset ...') # 获取MNIST/CIFAR10的验证集 if dataset == 'MNIST': _, valid_loader = get_mnist_train_validate_loader( dir_name='../data/MNIST/', batch_size=batch_size, valid_size=0.02, shuffle=True) else: _, valid_loader = get_cifar10_train_validate_loader( dir_name='../data/CIFAR10/', batch_size=batch_size, valid_size=0.02, shuffle=True) # radius:通过验证集得到最优的半径值 radius = nrc.search_best_radius(validation_loader=valid_loader, radius_min=args.radius_min, radius_max=args.radius_max, radius_step=args.radius_step) # 否则半径值为默认的0.01 else: radius = round(args.radius, 2) print( '######\nthe radius for NRC is set or searched as: {}\n######'.format( radius)) # 计算NRC模型在测试集上的分类精度 print( '\nStart to calculate the accuracy of region-based classification defense on testing dataset' ) raw_model.eval() total = 0.0 correct = 0.0 with torch.no_grad(): for images, labels in test_loader: nrc_labels = nrc.region_based_classification(samples=images, radius=radius, mean=args.mean, std=args.std) nrc_labels = torch.from_numpy(nrc_labels) total += labels.size(0) correct += (nrc_labels == labels).sum().item() ratio = correct / total print( '\nTest accuracy of the {} model on the testing dataset: {:.1f}/{:.1f} = {:.2f}%\n' .format(raw_model.model_name, correct, total, ratio * 100))
class SecurityEvaluate: def __init__(self, DataSet='MNIST', AttackName='LLC', AdvExamplesDir='../AdversarialExampleDatasets/', device=torch.device('cpu')): # DataSet:数据集名称 # dataset:数据集名称(大写) # AttackName:攻击名称 # attack_name:攻击名称(大写) # AdvExamplesDir:对抗性样本存放位置 self.device = device assert DataSet.upper() in ['MNIST', 'CIFAR10' ], "The data set must be MNIST or CIFAR10" self.dataset = DataSet.upper() # raw_model:加载模型 # ***********不同模型名称*********** raw_model_location = '{}{}/model/{}_raw.pt'.format( '../data/', self.dataset, self.dataset) if self.dataset == 'MNIST': self.raw_model = MNIST_CNN().to(device) self.raw_model.load(path=raw_model_location, device=device) else: self.raw_model = ResNet18().to(device) self.raw_model.load(path=raw_model_location, device=device) self.raw_model.eval() self.attack_name = AttackName.upper() supported_un_targeted = [ 'FGSM', 'RFGSM', 'BIM', 'PGD', 'DEEPFOOL', 'UAP' ] supported_targeted = ['LLC', "RLLC", 'ILLC', 'JSMA', 'CW2'] assert self.attack_name in supported_un_targeted or self.attack_name in supported_targeted, \ "\nCurrently, our implementation support attacks of FGSM, RFGSM, BIM, UMIFGSM, DeepFool, LLC, RLLC, ILLC, TMIFGSM, JSMA, CW2,....\n" # 设置Targeted是目标攻击还是非目标攻击 if self.attack_name.upper() in supported_un_targeted: self.Targeted = False print('the # {} # attack is a kind of Un-targeted attacks'.format( self.attack_name)) else: self.Targeted = True print('the # {} # attack is a kind of Targeted attacks'.format( self.attack_name)) # adv_samples,adv_labels,true_labels:加载对抗性样本,对抗性样本标签和真实标签 self.adv_samples = np.load('{}{}/{}/{}_AdvExamples.npy'.format( AdvExamplesDir, self.attack_name, self.dataset, self.attack_name)).astype(np.float32) self.adv_labels = np.load('{}{}/{}/{}_AdvLabels.npy'.format( AdvExamplesDir, self.attack_name, self.dataset, self.attack_name)) self.true_labels = np.load('{}{}/{}/{}_TrueLabels.npy'.format( AdvExamplesDir, self.attack_name, self.dataset, self.attack_name)) # targets_samples:获取目标攻击的标签 if self.attack_name.upper() in ['LLC', 'RLLC', 'ILLC']: self.targets_samples = np.load('{}{}/{}_llc.npy'.format( '../clean_datasets/', self.dataset, self.dataset)) else: self.targets_samples = np.load('{}{}/{}_targets.npy'.format( '../clean_datasets/', self.dataset, self.dataset)) # 对对抗性样本经过防御后的标签预测 def defense_predication(self, DefenseModelDirs, defense_name, **kwargs): # DefenseModelDirs:防御模型所在位置 # defense_name:防御名称(大写) re_train_defenses = { 'NAT', 'RLT', 'RLT1', 'RLT2', 'RLT3', 'EAT', 'UAPAT', 'NEAT', 'NRC', 'RAT', 'RAT1', 'RAT2', 'RAT3', 'RAT4', 'RAT5', 'RAT6', 'RAT7', 'RAT8', 'RAT9', 'RAT10', 'RAT11', 'MART', 'NEW_MART', 'NEW_MART1', 'NEW_MMA' } other_defenses = {'NRC'} defense_name = defense_name.upper().strip() assert defense_name in re_train_defenses or input_transformation_defenses or other_defenses # 如果是重新训练网络防御 if defense_name in re_train_defenses: print( '\n##{}## defense is a kind of complete defenses that retrain the model' .format(defense_name)) # 加载防御模型 defended_model_location = '{}/{}/{}_{}_enhanced.pt'.format( DefenseModelDirs, defense_name, self.dataset, defense_name) defended_model = MNIST_CNN().to( self.device) if self.dataset == 'MNIST' else ResNet18().to( self.device) defended_model.load(path=defended_model_location, device=self.device) defended_model.eval() # 进行标签预测 predication = predict(model=defended_model, samples=self.adv_samples, device=self.device) # 返回标签行向量 labels = torch.argmax(predication, 1).cpu().numpy() return labels else: if defense_name == 'NRC': print( '\n##{}## defense is a kind of region-based classification defenses ... ' .format(defense_name)) from Defenses.DefenseMethods.NRC import NRCDefense num_points = 1000 assert 'nrc_radius' in kwargs assert 'nrc_mean' in kwargs assert 'nrc_std' in kwargs radius = kwargs['nrc_radius'] mean = kwargs['nrc_mean'] std = kwargs['nrc_std'] nrc = NRCDefense(model=self.raw_model, defense_name='NRC', dataset=self.dataset, device=self.device, num_points=num_points) labels = nrc.region_based_classification( samples=self.adv_samples, radius=radius, mean=mean, std=std) return labels else: raise ValueError('{} is not supported!!!'.format(defense_name)) def success_rate(self, defense_predication): # defense_predication:防御预测结果标签 # adv_labels:对抗性样本标签 # true_labels:真实标签 true_labels = np.argmax(self.true_labels, 1) # targets:目标攻击的标签 targets = np.argmax(self.targets_samples, 1) assert defense_predication.shape == true_labels.shape and true_labels.shape == self.adv_labels.shape and self.adv_labels.shape == targets.shape original_misclassification = 0.0 defense_success = 0.0 for i in range(len(defense_predication)): # 如果为目标攻击,计算攻击成功条件下,防御成功的数量 if self.Targeted: if self.adv_labels[i] == targets[i]: original_misclassification += 1 if defense_predication[i] == true_labels[i]: defense_success += 1 # 如果为非目标攻击,计算攻击成功条件下,防御成功的数量 else: if self.adv_labels[i] != true_labels[i]: original_misclassification += 1 if defense_predication[i] == true_labels[i]: defense_success += 1 # 返回攻击成功和防御成功的数量 return original_misclassification, defense_success