def main(): parser = argparse.ArgumentParser() parser.add_argument('--file', type=str, required=True) parser.add_argument('--param', type=str, required=True) parser.add_argument('--batch_size', type=int, default=128) parser.add_argument('--data_path', type=str, default='data') parser.add_argument('--output_path', type=str, default='results') parser.add_argument('--random_state', type=int, default=1234) args = parser.parse_args() print(args) set_seeds(args.random_state) with open(args.param) as param_json: param = json.load(param_json) print('Param:', param) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('Device:', device) # eg: cifar10_resnet_baard_train_surrodata_eps2.0_size2000.pt name_parser = args.file.split('_') data = name_parser[0] model_name = name_parser[1] get_baard_output(data, model_name, args.data_path, args.output_path, args.file, param, args.batch_size, device)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--data', type=str, required=True, choices=['mnist', 'cifar10']) parser.add_argument('--model', type=str, required=True, choices=['basic', 'resnet', 'vgg']) parser.add_argument('--n_samples', type=int, default=2000) parser.add_argument('--eps', type=float, default=2.) parser.add_argument('--test', type=int, default=0, choices=[0, 1]) parser.add_argument('--batch_size', type=int, default=128) parser.add_argument('--data_path', type=str, default='data') parser.add_argument('--output_path', type=str, default='results') parser.add_argument('--random_state', type=int, default=1234) args = parser.parse_args() print(args) set_seeds(args.random_state) if not os.path.exists(args.output_path): print('Output folder does not exist. Create:', args.output_path) os.mkdir(args.output_path) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('Device:', device) time_start = time.time() train_adv(args.data, args.model, args.n_samples, args.eps, args.output_path, args.data_path, args.test, args.batch_size, device) time_elapsed = time.time() - time_start print('Total training time:', str(datetime.timedelta(seconds=time_elapsed))) print()
def main(): with open('data.json') as data_json: data_params = json.load(data_json) parser = argparse.ArgumentParser() parser.add_argument('--data', type=str, required=True) parser.add_argument('--model', type=str, default='svm', choices=['svm', 'tree']) parser.add_argument('--data_path', type=str, default='data') parser.add_argument('--output_path', type=str, default='results') parser.add_argument('--adv', type=str, default='fgsm_0.2') parser.add_argument('--param', type=str, default=None) parser.add_argument('--random_state', type=int, default=1234) args = parser.parse_args() set_seeds(args.random_state) if args.param == None: args.param = os.path.join('params', 'rc_param_{}.json'.format(args.data)) adv_root = '{}_{}_{}'.format(args.data, args.model, args.adv) print('Dataset:', args.data) print('Model:', args.model) print('Pretrained samples:', adv_root + '_adv.npy') with open(args.param) as param_json: param = json.load(param_json) param['n_classes'] = data_params['data'][args.data]['n_classes'] print('Param:', param) # Prepare data data_path = os.path.join(args.data_path, data_params['data'][args.data]['file_name']) print('Read file: {}'.format(data_path)) X, y = load_csv(data_path) # Apply scaling scaler = MinMaxScaler().fit(X) X = scaler.transform(X) n_test = data_params['data'][args.data]['n_test'] random_state = args.random_state X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=n_test, random_state=random_state) # Train model if args.model == 'svm': model = SVC(kernel="linear", C=1.0, gamma="scale", random_state=random_state) elif args.model == 'tree': model = ExtraTreeClassifier() else: raise NotImplementedError model.fit(X_train, y_train) acc_train = model.score(X_train, y_train) acc_test = model.score(X_test, y_test) print(('Train Acc: {:.4f}, ' + 'Test Acc: {:.4f}').format(acc_train, acc_test)) # Load adversarial examples path_x = os.path.join(args.output_path, '{}_x.npy'.format(adv_root)) path_y = os.path.join(args.output_path, '{}_y.npy'.format(adv_root)) path_adv = os.path.join(args.output_path, '{}_adv.npy'.format(adv_root)) X_benign = np.load(path_x) y_true = np.load(path_y) adv = np.load(path_adv) print(X_benign.shape, y_true.shape, adv.shape) print('Acc on clean:', model.score(X_benign, y_true)) print('Acc on adv:', model.score(adv, y_true)) n = len(X_benign) // 2 X_val = X_benign[n:] y_val = y_true[n:] # Train defence time_start = time.time() detector = SklearnRegionBasedClassifier( model=model, r=0.2, sample_size=1000, n_classes=param['n_classes'], x_min=0.0, x_max=1.0, r0=0.0, step_size=0.02, stop_value=0.4) r_best = detector.search_thresholds(X_val, model.predict(X_val), np.zeros_like(y_val), verbose=0) time_elapsed = time.time() - time_start print('Total training time:', str(datetime.timedelta(seconds=time_elapsed))) param = { "r": r_best, "sample_size": 1000, "batch_size": 512, "r0": 0, "step_size": 0.02, "stop_value": 0.40 } path_json = os.path.join('params', 'rc_param_{}_{}.json'.format(args.data, args.model)) with open(path_json, 'w') as f: json.dump(param, f) print('Save to:', path_json) print()
def main(): with open('data.json') as data_json: data_params = json.load(data_json) parser = argparse.ArgumentParser() parser.add_argument('--data', type=str, required=True, choices=data_params['datasets']) parser.add_argument('--model', type=str, default='svm', choices=['svm', 'tree']) parser.add_argument('--data_path', type=str, default='data') parser.add_argument('--output_path', type=str, default='results') parser.add_argument('--attack', type=str, required=True, choices=ATTACKS) parser.add_argument('--eps', type=float, default=0.3) # NOTE: In CW_L2 attack, eps is the upper bound of c. parser.add_argument('--batch_size', type=int, default=128) parser.add_argument('--n_samples', type=int, default=2000) parser.add_argument('--random_state', type=int, default=1234) args = parser.parse_args() print(args) set_seeds(args.random_state) if not os.path.exists(args.output_path): print('Output folder does not exist. Create:', args.output_path) os.mkdir(args.output_path) print('Dataset:', args.data) print('Running attack:', args.attack) # Prepare data data_path = os.path.join(args.data_path, data_params['data'][args.data]['file_name']) print('Read file: {}'.format(data_path)) X, y = load_csv(data_path) # Apply scaling scaler = MinMaxScaler().fit(X) X = scaler.transform(X) n_test = data_params['data'][args.data]['n_test'] random_state = args.random_state X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=n_test, random_state=random_state) # Train model if args.model == 'svm': model = SVC(kernel="linear", C=1.0, gamma="scale", random_state=random_state) elif args.model == 'tree': model = ExtraTreeClassifier(random_state=random_state) else: raise NotImplementedError model.fit(X_train, y_train) acc_train = model.score(X_train, y_train) acc_test = model.score(X_test, y_test) print(('Train Acc: {:.4f}, ' + 'Test Acc: {:.4f}').format(acc_train, acc_test)) # Get perfect subset pred_test = model.predict(X_test) idx_correct = np.where(pred_test == y_test)[0] X_test = X_test[idx_correct] y_test = y_test[idx_correct] classifier = SklearnClassifier(model=model, clip_values=(0.0, 1.0)) if args.attack == 'bim': eps_step = args.eps / 10.0 attack = BasicIterativeMethod( estimator=classifier, eps=args.eps, eps_step=eps_step, max_iter=100, targeted=False, batch_size=args.batch_size) elif args.attack == 'boundary': attack = BoundaryAttack( estimator=classifier, max_iter=1000, sample_size=20, targeted=False) elif args.attack == 'fgsm': attack = FastGradientMethod( estimator=classifier, eps=args.eps, batch_size=args.batch_size) elif args.attack == 'tree': attack = DecisionTreeAttack( classifier=classifier) else: raise NotImplementedError # How many examples do we have? if len(X_test) > args.n_samples: n = args.n_samples else: n = len(X_test) X_benign = X_test[:n] y_true = y_test[:n] adv = attack.generate(X_benign) acc = model.score(adv, y_true) print('Acc on adv:', acc) output_file = '{}_{}_{}_{}'.format(args.data, args.model, args.attack, str(args.eps)) path_x = os.path.join(args.output_path, '{}_x.npy'.format(output_file)) path_y = os.path.join(args.output_path, '{}_y.npy'.format(output_file)) path_adv = os.path.join(args.output_path, '{}_adv.npy'.format(output_file)) np.save(path_x, X_benign) np.save(path_y, y_true) np.save(path_adv, adv) print('Saved to:', path_adv) print()
def main(): with open('data.json') as data_json: data_params = json.load(data_json) parser = argparse.ArgumentParser() parser.add_argument('--data', type=str, required=True) parser.add_argument('--model', type=str, default='svm', choices=['svm', 'tree']) parser.add_argument('--data_path', type=str, default='data') parser.add_argument('--output_path', type=str, default='results') parser.add_argument('--random_state', type=int, default=1234) args = parser.parse_args() print(args) set_seeds(args.random_state) if not os.path.exists(args.output_path): print('Output folder does not exist. Create:', args.output_path) os.mkdir(args.output_path) print('data:', args.data) print('model:', args.model) # Prepare data data_path = os.path.join(args.data_path, data_params['data'][args.data]['file_name']) print('Read file: {}'.format(data_path)) X, y = load_csv(data_path) # Apply scaling scaler = MinMaxScaler().fit(X) X = scaler.transform(X) n_test = data_params['data'][args.data]['n_test'] random_state = args.random_state X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=n_test, random_state=random_state) # Train model if args.model == 'svm': model = SVC(kernel="linear", C=1.0, gamma="scale", random_state=random_state) elif args.model == 'tree': model = ExtraTreeClassifier(random_state=random_state) else: raise NotImplementedError model.fit(X_train, y_train) acc_train = model.score(X_train, y_train) acc_test = model.score(X_test, y_test) print(('Train Acc: {:.4f}, ' + 'Test Acc: {:.4f}').format(acc_train, acc_test)) # Get perfect subset pred_test = model.predict(X_test) idx_correct = np.where(pred_test == y_test)[0] X_test = X_test[idx_correct] y_test = y_test[idx_correct] X_baard = baard_preprocess(X_train) obj = { 'X_train': X_baard, 'y_train': y_train } path_ouput = os.path.join(args.output_path, '{}_{}_baard_train.pt'.format(args.data, args.model, args.model)) torch.save(obj, path_ouput) print('Save to:', path_ouput) print()
def main(seed, dataset_name, clf_name, detector_name, epsilon_lst,input_shape): set_seeds(SEEDS[seed]) device = device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('device:', device) # Load classifier print("load the classifier") file_model = os.path.join('result_{:}'.format(seed), '{:}_{:}_model.pt'.format(dataset_name, clf_name)) if clf_name == 'dnn': model = BaseModel(use_prob=False).to(device) elif clf_name == 'resnet': model = Resnet(use_prob=False).to(device) else: raise ValueError("model idx unknown") model.load_state_dict(torch.load(file_model, map_location=device)) file_data = os.path.join('result_{:}'.format(seed), '{:}_{:}_apgd2_2000.pt'.format(dataset_name, clf_name)) obj = torch.load(file_data) X = obj['X'] y = obj['y'] adv = obj['adv'] pred = predict_numpy(model, X, device) print('Acc on clean:', np.mean(pred == y)) pred = predict_numpy(model, adv, device) print('Acc on adv (epsilon 2):', np.mean(pred == y)) # Split data X_att_test = X[2000:3000] y_att_test = y[2000:3000] print("x attr shape ", X_att_test.shape) ######################################################################### # Load baard print("Load baard") file_baard_train = os.path.join( 'result_{:}'.format(seed), '{:}_{:}_baard_s1_train_data.pt'.format( dataset_name, clf_name)) obj = torch.load(file_baard_train) X_baard_train_s1 = obj['X_s1'] X_baard_train = obj['X'] y_baard_train = obj['y'] stages = [] stages.append(ApplicabilityStage(n_classes=10, quantile=1., verbose=False)) stages.append(ReliabilityStage(n_classes=10, k=10, quantile=1., verbose=False)) stages.append(DecidabilityStage(n_classes=10, k=100, quantile=1., verbose=False)) detector = BAARDOperator(stages=stages) detector.stages[0].fit(X_baard_train_s1, y_baard_train) for stage in detector.stages[1:]: stage.fit(X_baard_train, y_baard_train) print("load baard's thresholds") file_baard_threshold = os.path.join( 'result_{:}'.format(seed), '{:}_{:}_baard_threshold.pt'.format( dataset_name, clf_name)) thresholds = torch.load(file_baard_threshold)['thresholds'] detector.load(file_baard_threshold) print("load the surrogate") file_surro = os.path.join('result_{:}'.format(seed), '{:}_{:}_baard_surrogate.pt'.format( dataset_name, clf_name)) surrogate = get_pretrained_surrogate(file_surro, device) loss = torch.nn.CrossEntropyLoss() optimizer_clf = torch.optim.SGD( model.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4) art_classifier = PyTorchClassifier( model=model, loss=loss, input_shape=input_shape, nb_classes=10, optimizer=optimizer_clf ) optimizer_sur = torch.optim.SGD( surrogate.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4) art_detector = PyTorchClassifier( model=surrogate, loss=loss, input_shape=input_shape, nb_classes=2, optimizer=optimizer_sur ) clip_fun = BAARD_Clipper(detector) ######################################################################### pred_folder = 'result_{:}/predictions_wb_eval/{:}_{:}_{:}'.format(seed, dataset_name, clf_name, detector_name) print("compute prediction for samples at epsilon 0") x = X_att_test[:10] y = y_att_test[:10] # compute and save predictions cmpt_and_save_predictions(model, art_detector, detector, device, x, y, pred_folder, 0) for eps in epsilon_lst: print("epsilon ", eps) if dataset_name == 'mnist': loss_multiplier = 1. / 36. else: loss_multiplier = 0.1 attack = AutoProjectedGradientDescentDetectors( estimator=art_classifier, detector=art_detector, detector_th=0, detector_clip_fun=clip_fun, loss_type='logits_difference', batch_size=128, norm=2, eps=eps, eps_step=0.9, beta=0.5, max_iter=100) adv_x = attack.generate(x=x, y=None) # compute and save predictions cmpt_and_save_predictions(model, art_detector, detector, device, adv_x, y, pred_folder, eps)
def main(): with open('data.json') as data_json: data_params = json.load(data_json) parser = argparse.ArgumentParser() parser.add_argument('--data', type=str, required=True) parser.add_argument('--data_path', type=str, default='data') parser.add_argument('--output_path', type=str, default='results') parser.add_argument('--pretrained', type=str, required=True) parser.add_argument('--adv', type=str, required=True, help="Example: 'mnist_basic_apgd_0.3'") parser.add_argument('--defence', type=str, required=True, choices=data_params['defences']) parser.add_argument('--param', type=str, required=True) parser.add_argument('--suffix', type=str) parser.add_argument('--random_state', type=int, default=1234) parser.add_argument('--save', type=int, default=1, choices=[0, 1]) args = parser.parse_args() print(args) set_seeds(args.random_state) if not os.path.exists(args.output_path): print('Output folder does not exist. Create:', args.output_path) os.mkdir(args.output_path) print('Dataset:', args.data) print('Pretrained model:', args.pretrained) print('Pretrained samples:', args.adv + '_adv.npy') print('Defence:', args.defence) with open(args.param) as param_json: param = json.load(param_json) param['n_classes'] = data_params['data'][args.data]['n_classes'] print('Param:', param) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('Device: {}'.format(device)) # Prepare data transforms = tv.transforms.Compose([tv.transforms.ToTensor()]) if args.data == 'mnist': dataset_train = datasets.MNIST(args.data_path, train=True, download=True, transform=transforms) dataset_test = datasets.MNIST(args.data_path, train=False, download=True, transform=transforms) elif args.data == 'cifar10': dataset_train = datasets.CIFAR10(args.data_path, train=True, download=True, transform=transforms) dataset_test = datasets.CIFAR10(args.data_path, train=False, download=True, transform=transforms) else: data_path = os.path.join(args.data_path, data_params['data'][args.data]['file_name']) print('Read file:', data_path) X, y = load_csv(data_path) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=data_params['data'][args.data]['n_test'], random_state=args.random_state) scaler = MinMaxScaler().fit(X_train) X_train = scaler.transform(X_train) X_test = scaler.transform(X_test) dataset_train = TensorDataset(torch.from_numpy(X_train).type(torch.float32), torch.from_numpy(y_train).type(torch.long)) dataset_test = TensorDataset(torch.from_numpy(X_test).type(torch.float32), torch.from_numpy(y_test).type(torch.long)) loader_train = DataLoader(dataset_train, batch_size=512, shuffle=False) loader_test = DataLoader(dataset_test, batch_size=512, shuffle=False) shape_train = get_shape(loader_train.dataset) shape_test = get_shape(loader_test.dataset) print('Train set:', shape_train) print('Test set:', shape_test) use_prob = True print('Using softmax layer:', use_prob) # Load model if args.data == 'mnist': model = BaseModel(use_prob=use_prob).to(device) model_name = 'basic' elif args.data == 'cifar10': model_name = args.pretrained.split('_')[1] if model_name == 'resnet': model = Resnet(use_prob=use_prob).to(device) elif model_name == 'vgg': model = Vgg(use_prob=use_prob).to(device) else: raise ValueError('Unknown model: {}'.format(model_name)) else: n_features = data_params['data'][args.data]['n_features'] n_classes = data_params['data'][args.data]['n_classes'] model = NumericModel(n_features, n_hidden=n_features * 4, n_classes=n_classes, use_prob=use_prob).to(device) model_name = 'basic' + str(n_features * 4) loss = nn.CrossEntropyLoss() pretrained_path = os.path.join(args.output_path, args.pretrained) model.load_state_dict(torch.load(pretrained_path, map_location=device)) _, acc_train = validate(model, loader_train, loss, device) _, acc_test = validate(model, loader_test, loss, device) print('Accuracy on train set: {:.4f}%'.format(acc_train * 100)) print('Accuracy on test set: {:.4f}%'.format(acc_test * 100)) # Create a subset which only contains recognisable samples. # The original train and test sets are no longer needed. tensor_train_X, tensor_train_y = get_correct_examples(model, dataset_train, device=device, return_tensor=True) dataset_train = TensorDataset(tensor_train_X, tensor_train_y) loader_train = DataLoader(dataset_train, batch_size=512, shuffle=True) _, acc_perfect = validate(model, loader_train, loss, device) print('Accuracy on {} filtered train set: {:.4f}%'.format(len(dataset_train), acc_perfect * 100)) tensor_test_X, tensor_test_y = get_correct_examples(model, dataset_test, device=device, return_tensor=True) dataset_test = TensorDataset(tensor_test_X, tensor_test_y) loader_test = DataLoader(dataset_test, batch_size=512, shuffle=False) _, acc_perfect = validate(model, loader_test, loss, device) print('Accuracy on {} filtered test set: {:.4f}%'.format(len(dataset_test), acc_perfect * 100)) # Load pre-trained adversarial examples path_benign = os.path.join(args.output_path, args.adv + '_x.npy') path_adv = os.path.join(args.output_path, args.adv + '_adv.npy') path_y = os.path.join(args.output_path, args.adv + '_y.npy') X_benign = np.load(path_benign) adv = np.load(path_adv) y_true = np.load(path_y) dataset = TensorDataset(torch.from_numpy(X_benign), torch.from_numpy(y_true)) loader = DataLoader(dataset, batch_size=512, shuffle=False) _, acc = validate(model, loader, loss, device) print('Accuracy on {} benign samples: {:.4f}%'.format(len(dataset), acc * 100)) dataset = TensorDataset(torch.from_numpy(adv), torch.from_numpy(y_true)) loader = DataLoader(dataset, batch_size=512, shuffle=False) _, acc = validate(model, loader, loss, device) print('Accuracy on {} adversarial examples: {:.4f}%'.format(len(dataset), acc * 100)) # Do NOT shuffle the indices, so different defences can use the same test set. dataset = TensorDataset(torch.from_numpy(adv)) loader = DataLoader(dataset, batch_size=512, shuffle=False) pred_adv = predict(model, loader, device).cpu().detach().numpy() # Find the thresholds using the 2nd half n = len(X_benign) // 2 # Merge benign samples and adversarial examples into one set. # This labels indicate a sample is an adversarial example or not. X_val, labels_val = merge_and_generate_labels(adv[n:], X_benign[n:], flatten=False) # The predictions for benign samples are exactly same as the true labels. pred_val = np.concatenate((pred_adv[n:], y_true[n:])) X_train = tensor_train_X.cpu().detach().numpy() y_train = tensor_train_y.cpu().detach().numpy() # Train defence time_start = time.time() if args.defence == 'baard': sequence = param['sequence'] stages = [] if sequence[0]: stages.append(ApplicabilityStage(n_classes=param['n_classes'], quantile=param['q1'])) if sequence[1]: stages.append(ReliabilityStage(n_classes=param['n_classes'], k=param['k_re'], quantile=param['q2'])) if sequence[2]: stages.append(DecidabilityStage(n_classes=param['n_classes'], k=param['k_de'], quantile=param['q3'])) print('BAARD: # of stages:', len(stages)) detector = BAARDOperator(stages=stages) # Run preprocessing baard_train_path = os.path.join(args.output_path, '{}_{}_baard_train.pt'.format(args.data, model_name)) obj = torch.load(baard_train_path) X_baard = obj['X_train'] y_train = obj['y_train'] # Fit the model with the filtered the train set. detector.stages[0].fit(X_baard, y_train) detector.stages[1].fit(X_train, y_train) if len(detector.stages) == 3: detector.stages[2].fit(X_train, y_train) detector.search_thresholds(X_val, pred_val, labels_val) path_baard = os.path.join(args.output_path, 'baard_{}_{}_param.pt'.format(args.data, model_name)) detector.save(path_baard) elif args.defence == 'fs': squeezers = [] if args.data == 'mnist': squeezers.append(DepthSqueezer(x_min=0.0, x_max=1.0, bit_depth=1)) squeezers.append(MedianSqueezer(x_min=0.0, x_max=1.0, kernel_size=2)) elif args.data == 'cifar10': squeezers.append(DepthSqueezer(x_min=0.0, x_max=1.0, bit_depth=4)) squeezers.append(MedianSqueezer(x_min=0.0, x_max=1.0, kernel_size=2)) squeezers.append(NLMeansColourSqueezer(x_min=0.0, x_max=1.0, h=2, templateWindowsSize=3, searchWindowSize=13)) else: raise NotImplementedError print('FS: # of squeezers:', len(squeezers)) detector = FeatureSqueezingTorch( classifier=model, lr=0.001, momentum=0.9, weight_decay=5e-4, loss=loss, batch_size=128, x_min=0.0, x_max=1.0, squeezers=squeezers, n_classes=param['n_classes'], device=device) path_fs = os.path.join(args.output_path, '{}_fs.pt'.format(args.pretrained.split('.')[0])) detector.load(path_fs) detector.search_thresholds(X_val, pred_val, labels_val) elif args.defence == 'lid': # This batch_size is not same as the mini batch size for the neural network. before_softmax = args.data == 'cifar10' detector = LidDetector( model, k=param['k'], batch_size=param['batch_size'], x_min=0.0, x_max=1.0, device=device, before_softmax=before_softmax) # LID uses different training set X_train, y_train = detector.get_train_set(X_benign[n:], adv[n:], std_dominator=param['std_dominator']) detector.fit(X_train, y_train, verbose=1) elif args.defence == 'magnet': magnet_detectors = [] # Different datasets require different autoencoders. if args.data == 'mnist': # autoencoder1 and autoencoder2 magnet_detectors.append(MagNetDetector( encoder=Autoencoder1(n_channel=1), classifier=model, lr=param['lr'], batch_size=param['batch_size'], weight_decay=param['weight_decay'], x_min=0.0, x_max=1.0, noise_strength=param['noise_strength'], algorithm='error', p=1, device=device)) magnet_detectors.append(MagNetDetector( encoder=Autoencoder2(n_channel=1), classifier=model, lr=param['lr'], batch_size=param['batch_size'], weight_decay=param['weight_decay'], x_min=0.0, x_max=1.0, noise_strength=param['noise_strength'], algorithm='error', p=2, device=device)) elif args.data == 'cifar10': autoencoder = Autoencoder2( n_channel=data_params['data'][args.data]['n_features'][0]) # There are 3 autoencoder based detectors, but they use the same architecture. magnet_detectors.append(MagNetDetector( encoder=autoencoder, classifier=model, lr=param['lr'], batch_size=param['batch_size'], weight_decay=param['weight_decay'], x_min=0.0, x_max=1.0, noise_strength=param['noise_strength'], algorithm='error', p=2, device=device)) magnet_detectors.append(MagNetDetector( encoder=autoencoder, classifier=model, lr=param['lr'], batch_size=param['batch_size'], weight_decay=param['weight_decay'], x_min=0.0, x_max=1.0, noise_strength=param['noise_strength'], algorithm='prob', temperature=10, device=device)) magnet_detectors.append(MagNetDetector( encoder=autoencoder, classifier=model, lr=param['lr'], batch_size=param['batch_size'], weight_decay=param['weight_decay'], x_min=0.0, x_max=1.0, noise_strength=param['noise_strength'], algorithm='prob', temperature=40, device=device)) else: raise ValueError('Magnet requires autoencoder.') for i, ae in enumerate(magnet_detectors, start=1): ae_path = os.path.join(args.output_path, 'autoencoder_{}_{}_{}.pt'.format(args.data, model_name, i)) ae.load(ae_path) tensor_X_test, _ = dataset2tensor(dataset_test) X_test = tensor_X_test.cpu().detach().numpy() print('Autoencoder {} MSE training set: {:.6f}, test set: {:.6f}'.format(i, ae.score(X_train), ae.score(X_test))) print('Autoencoder {} threshold: {}'.format(i, ae.threshold)) reformer = MagNetAutoencoderReformer( encoder=magnet_detectors[0].encoder, batch_size=param['batch_size'], device=device) detector = MagNetOperator( classifier=model, detectors=magnet_detectors, reformer=reformer, batch_size=param['batch_size'], device=device) elif args.defence == 'rc': detector = RegionBasedClassifier( model=model, r=param['r'], sample_size=param['sample_size'], n_classes=param['n_classes'], x_min=0.0, x_max=1.0, batch_size=param['batch_size'], r0=param['r0'], step_size=param['step_size'], stop_value=param['stop_value'], device=device) # Region-based classifier only uses benign samples to search threshold. # The r value is already set to the optimal. We don't need to search it. # detector.search_thresholds(X_val, pred_val, labels_val, verbose=0) else: raise ValueError('{} is not supported!'.format(args.defence)) time_elapsed = time.time() - time_start print('Total training time:', str(datetime.timedelta(seconds=time_elapsed))) # Test defence time_start = time.time() X_test, labels_test = merge_and_generate_labels(adv[:n], X_benign[:n], flatten=False) pred_test = np.concatenate((pred_adv[:n], y_true[:n])) y_test = np.concatenate((y_true[:n], y_true[:n])) # Only MegNet uses reformer. X_reformed = None if args.defence == 'magnet': X_reformed, res_test = detector.detect(X_test, pred_test) y_pred = predict_numpy(model, X_reformed, device) elif args.defence == 'rc': y_pred = detector.detect(X_test, pred_test) res_test = np.zeros_like(y_pred) else: res_test = detector.detect(X_test, pred_test) y_pred = pred_test acc = acc_on_adv(y_pred[:n], y_test[:n], res_test[:n]) if args.defence == 'rc': fpr = np.mean(y_pred[n:] != y_test[n:]) else: fpr = np.mean(res_test[n:]) print('Acc_on_adv:', acc) print('FPR:', fpr) time_elapsed = time.time() - time_start print('Total test time:', str(datetime.timedelta(seconds=time_elapsed))) # Save results suffix = '_' + args.suffix if args.suffix is not None else '' if args.save: path_result = os.path.join(args.output_path, '{}_{}{}.pt'.format(args.adv, args.defence, suffix)) torch.save({ 'X_val': X_val, 'y_val': np.concatenate((y_true[n:], y_true[n:])), 'labels_val': labels_val, 'X_test': X_test, 'y_test': y_test, 'labels_test': labels_test, 'res_test': y_pred if args.defence == 'rc' else res_test, 'X_reformed': X_reformed, 'param': param}, path_result) print('Saved to:', path_result) else: print('No file is save!') print()
def main(): with open('data.json') as data_json: data_params = json.load(data_json) parser = argparse.ArgumentParser() parser.add_argument('--data', type=str) parser.add_argument('--data_path', type=str, default='data') parser.add_argument('--output_path', type=str, default='results') parser.add_argument('--pretrained', type=str, required=True) parser.add_argument('--batch_size', type=int, default=128) parser.add_argument('--attack', type=str, required=True, choices=data_params['attacks']) parser.add_argument('--eps', type=float, default=0.3) # NOTE: In CW_L2 attack, eps is the upper bound of c. parser.add_argument('--n_samples', type=int, default=2000) parser.add_argument('--random_state', type=int, default=1234) args = parser.parse_args() print(args) set_seeds(args.random_state) if not os.path.exists(args.output_path): print('Output folder does not exist. Create:', args.output_path) os.mkdir(args.output_path) print('Dataset:', args.data) print('Pretrained model:', args.pretrained) print('Running attack: {}'.format(args.attack)) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('Device: {}'.format(device)) # Prepare data transforms = tv.transforms.Compose([tv.transforms.ToTensor()]) if args.data == 'mnist': dataset_train = datasets.MNIST(args.data_path, train=True, download=True, transform=transforms) dataset_test = datasets.MNIST(args.data_path, train=False, download=True, transform=transforms) elif args.data == 'cifar10': dataset_train = datasets.CIFAR10(args.data_path, train=True, download=True, transform=transforms) dataset_test = datasets.CIFAR10(args.data_path, train=False, download=True, transform=transforms) else: data_path = os.path.join(args.data_path, data_params['data'][args.data]['file_name']) print('Read file:', data_path) X, y = load_csv(data_path) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=data_params['data'][args.data]['n_test'], random_state=args.random_state) scaler = MinMaxScaler().fit(X_train) X_train = scaler.transform(X_train) X_test = scaler.transform(X_test) dataset_train = TensorDataset(torch.from_numpy(X_train).type(torch.float32), torch.from_numpy(y_train).type(torch.long)) dataset_test = TensorDataset(torch.from_numpy(X_test).type(torch.float32), torch.from_numpy(y_test).type(torch.long)) dataloader_train = DataLoader(dataset_train, 256, shuffle=False) dataloader_test = DataLoader(dataset_test, 256, shuffle=False) shape_train = get_shape(dataloader_train.dataset) shape_test = get_shape(dataloader_test.dataset) print('Train set:', shape_train) print('Test set:', shape_test) # Load model use_prob = args.attack not in ['apgd', 'apgd1', 'apgd2', 'cw2', 'cwinf'] print('Attack:', args.attack) print('Using softmax layer:', use_prob) if args.data == 'mnist': model = BaseModel(use_prob=use_prob).to(device) model_name = 'basic' elif args.data == 'cifar10': model_name = args.pretrained.split('_')[1] if model_name == 'resnet': model = Resnet(use_prob=use_prob).to(device) elif model_name == 'vgg': model = Vgg(use_prob=use_prob).to(device) else: raise ValueError('Unknown model: {}'.format(model_name)) else: n_features = data_params['data'][args.data]['n_features'] n_classes = data_params['data'][args.data]['n_classes'] model = NumericModel( n_features, n_hidden=n_features * 4, n_classes=n_classes, use_prob=use_prob).to(device) model_name = 'basic' + str(n_features * 4) optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4) loss = nn.CrossEntropyLoss() pretrained_path = os.path.join(args.output_path, args.pretrained) model.load_state_dict(torch.load(pretrained_path, map_location=device)) _, acc_train = validate(model, dataloader_train, loss, device) _, acc_test = validate(model, dataloader_test, loss, device) print('Accuracy on train set: {:.4f}%'.format(acc_train * 100)) print('Accuracy on test set: {:.4f}%'.format(acc_test * 100)) # Create a subset which only contains recognisable samples. tensor_test_X, tensor_test_y = get_correct_examples( model, dataset_test, device=device, return_tensor=True) dataset_perfect = TensorDataset(tensor_test_X, tensor_test_y) loader_perfect = DataLoader(dataset_perfect, batch_size=512, shuffle=False) _, acc_perfect = validate(model, loader_perfect, loss, device) print('Accuracy on {} filtered test examples: {:.4f}%'.format( len(dataset_perfect), acc_perfect * 100)) # Generate adversarial examples n_features = data_params['data'][args.data]['n_features'] n_classes = data_params['data'][args.data]['n_classes'] if isinstance(n_features, int): n_features = (n_features,) classifier = PyTorchClassifier( model=model, loss=loss, input_shape=n_features, optimizer=optimizer, nb_classes=n_classes, clip_values=(0.0, 1.0), device_type='gpu') if args.attack == 'apgd': eps_step = args.eps / 10.0 if args.eps <= 0.1 else 0.1 attack = AutoProjectedGradientDescent( estimator=classifier, eps=args.eps, eps_step=eps_step, max_iter=1000, batch_size=args.batch_size, targeted=False) elif args.attack == 'apgd1': attack = AutoProjectedGradientDescent( estimator=classifier, norm=1, eps=args.eps, eps_step=0.1, max_iter=1000, batch_size=args.batch_size, targeted=False) elif args.attack == 'apgd2': attack = AutoProjectedGradientDescent( estimator=classifier, norm=2, eps=args.eps, eps_step=0.1, max_iter=1000, batch_size=args.batch_size, targeted=False) elif args.attack == 'bim': eps_step = args.eps / 10.0 attack = BasicIterativeMethod( estimator=classifier, eps=args.eps, eps_step=eps_step, max_iter=1000, batch_size=args.batch_size, targeted=False) elif args.attack == 'boundary': attack = BoundaryAttack( estimator=classifier, max_iter=1000, sample_size=args.batch_size, targeted=False) elif args.attack == 'cw2': # NOTE: Do NOT increase the batch size! attack = CarliniWagnerAttackL2( model=model, n_classes=n_classes, confidence=args.eps, verbose=True, check_prob=False, batch_size=args.batch_size, targeted=False) elif args.attack == 'cwinf': attack = CarliniLInfMethod( classifier=classifier, confidence=args.eps, max_iter=1000, batch_size=args.batch_size, targeted=False) elif args.attack == 'deepfool': attack = DeepFool( classifier=classifier, epsilon=args.eps, batch_size=args.batch_size) elif args.attack == 'fgsm': attack = FastGradientMethod( estimator=classifier, eps=args.eps, batch_size=args.batch_size) elif args.attack == 'jsma': attack = SaliencyMapMethod( classifier=classifier, gamma=args.eps, batch_size=args.batch_size) elif args.attack == 'line': if args.data == 'mnist': color = args.eps elif args.data == 'cifar10': color = (args.eps, args.eps, args.eps) else: raise NotImplementedError attack = LineAttack(color=color, thickness=1) elif args.attack == 'shadow': attack = ShadowAttack( estimator=classifier, batch_size=args.batch_size, targeted=False, verbose=False) elif args.attack == 'watermark': attack = WaterMarkAttack( eps=args.eps, n_classes=data_params['data'][args.data]['n_classes'], x_min=0.0, x_max=1.0, targeted=False) X_train, y_train = get_correct_examples(model, dataset_train, device=device, return_tensor=True) X_train = X_train.cpu().detach().numpy() y_train = y_train.cpu().detach().numpy() attack.fit(X_train, y_train) else: raise NotImplementedError if len(dataset_perfect) > args.n_samples: n = args.n_samples else: n = len(dataset_perfect) X_benign = tensor_test_X[:n].cpu().detach().numpy() y = tensor_test_y[:n].cpu().detach().numpy() print('Creating {} adversarial examples with eps={} (Not all attacks use eps)'.format(n, args.eps)) time_start = time.time() # Shadow attack only takes single sample! if args.attack == 'shadow': adv = np.zeros_like(X_benign) for i in trange(len(X_benign)): adv[i] = attack.generate(x=np.expand_dims(X_benign[i], axis=0)) elif args.attack == 'watermark': # This is untargeted. adv = attack.generate(X_benign, y) else: adv = attack.generate(x=X_benign) time_elapsed = time.time() - time_start print('Total time spend: {}'.format(str(datetime.timedelta(seconds=time_elapsed)))) pred_benign = np.argmax(classifier.predict(X_benign), axis=1) acc_benign = np.sum(pred_benign == y) / n pred_adv = np.argmax(classifier.predict(adv), axis=1) acc_adv = np.sum(pred_adv == y) / n print("Accuracy on benign samples: {:.4f}%".format(acc_benign * 100)) print("Accuracy on adversarial examples: {:.4f}%".format(acc_adv * 100)) # Save results if args.n_samples < 2000: output_file = '{}_{}_{}_{}_size{}'.format(args.data, model_name, args.attack, str(args.eps), args.n_samples) else: output_file = '{}_{}_{}_{}'.format(args.data, model_name, args.attack, str(args.eps)) path_x = os.path.join(args.output_path, '{}_x.npy'.format(output_file)) path_y = os.path.join(args.output_path, '{}_y.npy'.format(output_file)) path_adv = os.path.join(args.output_path, '{}_adv.npy'.format(output_file)) np.save(path_x, X_benign) np.save(path_y, y) np.save(path_adv, adv) print('Saved to:', '{}_adv.npy'.format(output_file)) print()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--data', type=str, required=True, choices=['mnist', 'cifar10']) parser.add_argument('--data_path', type=str, default='data') parser.add_argument('--output_path', type=str, default='results') parser.add_argument('--pretrained', type=str, required=True) parser.add_argument('--param', type=str, required=True) parser.add_argument('--random_state', type=int, default=1234) args = parser.parse_args() print(args) set_seeds(args.random_state) if not os.path.exists(args.output_path): print('Output folder does not exist. Create:', args.output_path) os.mkdir(args.output_path) print('Dataset:', args.data) print('Pretrained model:', args.pretrained) with open(args.param) as param_json: param = json.load(param_json) param['n_classes'] = 10 print('Param:', param) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('Device: {}'.format(device)) # Prepare data transforms = tv.transforms.Compose([tv.transforms.ToTensor()]) if args.data == 'mnist': dataset_train = datasets.MNIST(args.data_path, train=True, download=True, transform=transforms) dataset_test = datasets.MNIST(args.data_path, train=False, download=True, transform=transforms) elif args.data == 'cifar10': dataset_train = datasets.CIFAR10(args.data_path, train=True, download=True, transform=transforms) dataset_test = datasets.CIFAR10(args.data_path, train=False, download=True, transform=transforms) else: raise ValueError('{} is not supported.'.format(args.data)) # Note: Train set alway shuffle! loader_train = DataLoader(dataset_train, batch_size=512, shuffle=True) loader_test = DataLoader(dataset_test, batch_size=512, shuffle=False) shape_train = get_shape(loader_train.dataset) shape_test = get_shape(loader_test.dataset) print('Train set:', shape_train) print('Test set:', shape_test) use_prob = True print('Using softmax layer:', use_prob) # Load model if args.data == 'mnist': model = BaseModel(use_prob=use_prob).to(device) model_name = 'basic' else: # args.data == 'cifar10': model_name = args.pretrained.split('_')[1] if model_name == 'resnet': model = Resnet(use_prob=use_prob).to(device) elif model_name == 'vgg': model = Vgg(use_prob=use_prob).to(device) else: raise ValueError('Unknown model: {}'.format(model_name)) loss = nn.CrossEntropyLoss() pretrained_path = os.path.join(args.output_path, args.pretrained) model.load_state_dict(torch.load(pretrained_path)) _, acc_train = validate(model, loader_train, loss, device) _, acc_test = validate(model, loader_test, loss, device) print('Accuracy on train set: {:.4f}%'.format(acc_train * 100)) print('Accuracy on test set: {:.4f}%'.format(acc_test * 100)) tensor_train_X, tensor_train_y = dataset2tensor(dataset_train) X_train = tensor_train_X.cpu().detach().numpy() y_train = tensor_train_y.cpu().detach().numpy() # Train defence squeezers = [] if args.data == 'mnist': squeezers.append(DepthSqueezer(x_min=0.0, x_max=1.0, bit_depth=1)) squeezers.append(MedianSqueezer(x_min=0.0, x_max=1.0, kernel_size=2)) else: # CIFAR10 squeezers.append(DepthSqueezer(x_min=0.0, x_max=1.0, bit_depth=4)) squeezers.append(MedianSqueezer(x_min=0.0, x_max=1.0, kernel_size=2)) squeezers.append(NLMeansColourSqueezer(x_min=0.0, x_max=1.0, h=2, templateWindowsSize=3, searchWindowSize=13)) print('FS: # of squeezers:', len(squeezers)) detector = FeatureSqueezingTorch( classifier=model, lr=0.001, momentum=0.9, weight_decay=5e-4, loss=loss, batch_size=128, x_min=0.0, x_max=1.0, squeezers=squeezers, n_classes=param['n_classes'], device=device) detector.fit(X_train, y_train, epochs=param['epochs'], verbose=1) path_fs = os.path.join(args.output_path, '{}_fs.pt'.format(args.pretrained.split('.')[0])) detector.save(path_fs) print('Saved fs to:', path_fs) print()
def main(): with open('data.json') as data_json: data_params = json.load(data_json) parser = argparse.ArgumentParser() parser.add_argument('--data', type=str, required=True) parser.add_argument('--data_path', type=str, default='data') parser.add_argument('--output_path', type=str, default='results') parser.add_argument('--pretrained', type=str, required=True) parser.add_argument('--adv', type=str, required=True, help="Example: 'mnist_basic_apgd_0.3'") parser.add_argument('--random_state', type=int, default=1234) args = parser.parse_args() print(args) set_seeds(args.random_state) print('Dataset:', args.data) print('Pretrained model:', args.pretrained) print('Pretrained samples:', args.adv + '_adv.npy') device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('Device: {}'.format(device)) # Prepare data transforms = tv.transforms.Compose([tv.transforms.ToTensor()]) if args.data == 'mnist': dataset_train = datasets.MNIST(args.data_path, train=True, download=True, transform=transforms) dataset_test = datasets.MNIST(args.data_path, train=False, download=True, transform=transforms) elif args.data == 'cifar10': dataset_train = datasets.CIFAR10(args.data_path, train=True, download=True, transform=transforms) dataset_test = datasets.CIFAR10(args.data_path, train=False, download=True, transform=transforms) else: data_path = os.path.join(args.data_path, data_params['data'][args.data]['file_name']) print('Read file:', data_path) X, y = load_csv(data_path) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=data_params['data'][args.data]['n_test'], random_state=args.random_state) scaler = MinMaxScaler().fit(X_train) X_train = scaler.transform(X_train) X_test = scaler.transform(X_test) dataset_train = TensorDataset( torch.from_numpy(X_train).type(torch.float32), torch.from_numpy(y_train).type(torch.long)) dataset_test = TensorDataset( torch.from_numpy(X_test).type(torch.float32), torch.from_numpy(y_test).type(torch.long)) # Note: Train set alway shuffle! loader_train = DataLoader(dataset_train, batch_size=512, shuffle=True) loader_test = DataLoader(dataset_test, batch_size=512, shuffle=False) shape_train = get_shape(loader_train.dataset) shape_test = get_shape(loader_test.dataset) print('Train set:', shape_train) print('Test set:', shape_test) use_prob = True print('Using softmax layer:', use_prob) n_classes = data_params['data'][args.data]['n_classes'] # Load model if args.data == 'mnist': model = BaseModel(use_prob=use_prob).to(device) model_name = 'basic' elif args.data == 'cifar10': model_name = args.pretrained.split('_')[1] if model_name == 'resnet': model = Resnet(use_prob=use_prob).to(device) elif model_name == 'vgg': model = Vgg(use_prob=use_prob).to(device) else: raise ValueError('Unknown model: {}'.format(model_name)) else: n_features = data_params['data'][args.data]['n_features'] model = NumericModel(n_features, n_hidden=n_features * 4, n_classes=n_classes, use_prob=use_prob).to(device) model_name = 'basic' + str(n_features * 4) loss = nn.CrossEntropyLoss() pretrained_path = os.path.join(args.output_path, args.pretrained) model.load_state_dict(torch.load(pretrained_path)) _, acc_train = validate(model, loader_train, loss, device) _, acc_test = validate(model, loader_test, loss, device) print('Accuracy on train set: {:.4f}%'.format(acc_train * 100)) print('Accuracy on test set: {:.4f}%'.format(acc_test * 100)) # Create a subset which only contains recognisable samples. # The original train and test sets are no longer needed. tensor_train_X, tensor_train_y = get_correct_examples(model, dataset_train, device=device, return_tensor=True) dataset_train = TensorDataset(tensor_train_X, tensor_train_y) loader_train = DataLoader(dataset_train, batch_size=512, shuffle=True) _, acc_perfect = validate(model, loader_train, loss, device) print('Accuracy on {} filtered train set: {:.4f}%'.format( len(dataset_train), acc_perfect * 100)) tensor_test_X, tensor_test_y = get_correct_examples(model, dataset_test, device=device, return_tensor=True) dataset_test = TensorDataset(tensor_test_X, tensor_test_y) loader_test = DataLoader(dataset_test, batch_size=512, shuffle=False) _, acc_perfect = validate(model, loader_test, loss, device) print('Accuracy on {} filtered test set: {:.4f}%'.format( len(dataset_test), acc_perfect * 100)) # Load pre-trained adversarial examples path_benign = os.path.join(args.output_path, args.adv + '_x.npy') path_adv = os.path.join(args.output_path, args.adv + '_adv.npy') path_y = os.path.join(args.output_path, args.adv + '_y.npy') X_benign = np.load(path_benign) adv = np.load(path_adv) y_true = np.load(path_y) dataset = TensorDataset(torch.from_numpy(X_benign), torch.from_numpy(y_true)) loader = DataLoader(dataset, batch_size=512, shuffle=False) _, acc = validate(model, loader, loss, device) print('Accuracy on {} benign samples: {:.4f}%'.format( len(dataset), acc * 100)) dataset = TensorDataset(torch.from_numpy(adv), torch.from_numpy(y_true)) loader = DataLoader(dataset, batch_size=512, shuffle=False) _, acc = validate(model, loader, loss, device) print('Accuracy on {} adversarial examples: {:.4f}%'.format( len(dataset), acc * 100)) # Do NOT shuffle the indices, so different defences can use the same test set. dataset = TensorDataset(torch.from_numpy(adv)) loader = DataLoader(dataset, batch_size=512, shuffle=False) pred_adv = predict(model, loader, device).cpu().detach().numpy() # Find the thresholds using the 2nd half n = len(X_benign) // 2 # Merge benign samples and adversarial examples into one set. # This labels indicate a sample is an adversarial example or not. X_val, labels_val = merge_and_generate_labels(adv[n:], X_benign[n:], flatten=False) # The predictions for benign samples are exactly same as the true labels. pred_val = np.concatenate((pred_adv[n:], y_true[n:])) X_train = tensor_train_X.cpu().detach().numpy() y_train = tensor_train_y.cpu().detach().numpy() # Train defence time_start = time.time() detector = RegionBasedClassifier(model=model, r=0.2, sample_size=1000, n_classes=n_classes, x_min=0.0, x_max=1.0, batch_size=512, r0=0.0, step_size=0.02, stop_value=0.4, device=device) r_best = detector.search_thresholds(X_val, pred_val, labels_val, verbose=0) time_elapsed = time.time() - time_start print('Total training time:', str(datetime.timedelta(seconds=time_elapsed))) param = { "r": r_best, "sample_size": 1000, "batch_size": 512, "r0": 0, "step_size": 0.02, "stop_value": 0.40 } path_json = os.path.join( 'params', 'rc_param_{}_{}.json'.format(args.data, args.model)) with open(path_json, 'w') as f: json.dump(param, f) print('Save to:', path_json) print()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--data', type=str, required=True, choices=['mnist', 'cifar10']) parser.add_argument('--model', type=str, required=True, choices=['basic', 'resnet', 'vgg']) parser.add_argument('--train', type=str, required=True) parser.add_argument('--test', type=str, required=True) parser.add_argument('--epochs', type=int, default=200) parser.add_argument('--batch_size', type=int, default=128) parser.add_argument('--data_path', type=str, default='results') parser.add_argument('--output_path', type=str, default='results') parser.add_argument('--random_state', type=int, default=1234) args = parser.parse_args() print(args) set_seeds(args.random_state) if not os.path.exists(args.output_path): print('Output folder does not exist. Create:', args.output_path) os.mkdir(args.output_path) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('Device:', device) n_channels = 1 if args.data == 'mnist' else 3 # Load training data (Shuffle is required) path_train = os.path.join(args.data_path, args.train) obj_train = torch.load(path_train) X_train = np.concatenate((obj_train['X'], obj_train['adv'])) y_train = np.concatenate( (obj_train['baard_label_x'], obj_train['baard_label_adv'])) dataset_train = TensorDataset( torch.from_numpy(X_train).type(torch.float32), torch.from_numpy(y_train).type(torch.long)) loader_train = DataLoader(dataset_train, batch_size=args.batch_size, shuffle=True) print('[Train set] Adv: {}, Benign: {}'.format(np.sum(y_train == 1), np.sum(y_train == 0))) # Load test data (Do not shuffle) path_test = os.path.join(args.data_path, args.test) obj_test = torch.load(path_test) X_test = np.concatenate((obj_test['X'], obj_test['adv'])) y_test = np.concatenate( (obj_test['baard_label_x'], obj_test['baard_label_adv'])) dataset_test = TensorDataset( torch.from_numpy(X_test).type(torch.float32), torch.from_numpy(y_test).type(torch.long)) loader_test = DataLoader(dataset_test, batch_size=args.batch_size, shuffle=False) print('[Test set] Adv: {}, Benign: {}'.format(np.sum(y_test == 1), np.sum(y_test == 0))) model = SurrogateModel(in_channels=n_channels, use_prob=True) time_start = time.time() train_surrogate(model, loader_train, loader_test, args.epochs, device) time_elapsed = time.time() - time_start print('Total training time:', str(datetime.timedelta(seconds=time_elapsed))) file_name = os.path.join( args.output_path, '{}_{}_surrogate_{}.pt'.format(args.data, args.model, args.epochs)) torch.save(model.state_dict(), file_name) print('Save to:', file_name) print()
def main(seed, dataset_name, clf_name, detector_name, epsilon_lst, input_shape, json_param, path): set_seeds(SEEDS[seed]) device = device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') print('device:', device) # Load classifier print("load the classifier") file_model = os.path.join( 'result_{:}'.format(seed), '{:}_{:}_model.pt'.format(dataset_name, clf_name)) if clf_name == 'dnn': model = BaseModel(use_prob=False).to(device) elif clf_name == 'resnet': model = Resnet(use_prob=False).to(device) else: raise ValueError("model idx unknown") model.load_state_dict(torch.load(file_model, map_location=device)) file_data = os.path.join( 'result_{:}'.format(seed), '{:}_{:}_apgd2_2000.pt'.format(dataset_name, clf_name)) obj = torch.load(file_data) X = obj['X'] y = obj['y'] adv = obj['adv'] print("undefended model acc") pred = predict_numpy(model, X, device) print('Acc on clean:', np.mean(pred == y)) # Split data X_att_test = X[2000:3000].astype(np.float32) y_att_test = y[2000:3000].astype(np.float32) print("x attr shape ", X_att_test.shape) ################################################################# print("Load Magnet") with open(json_param) as j: param = json.load(j) print("before load magnet") model_with_reformer_nn_module, detector_nn_module, full_magnet_orig = \ loadmagnet(dataset_name, clf_name,param, device,path, model) print("Magnet loaded") loss = torch.nn.CrossEntropyLoss() # this one return the logits art_classifier = PyTorchClassifier(model=model_with_reformer_nn_module, loss=loss, input_shape=input_shape, nb_classes=10, optimizer=None) # y_pred = model_with_reformer_nn_module(X) # print("model_with_reformer_nn_module", y_pred.shape) y_pred = art_classifier.predict(X) print("art_classifier", y_pred.shape) print("check full magnet ") _, y_pred = full_magnet_orig.detect(X) print("full magnet", y_pred.shape) print("check detector nn module") # correcly return an array with the logits y_pred = detector_nn_module(X) print("y pred ", y_pred) print("detector_nn_module", y_pred.shape) print("create pytorch detector") # must be only the detector art_detector = PyTorchClassifier(model=detector_nn_module, loss=loss, input_shape=input_shape, nb_classes=2, optimizer=None) print("check art detector") y_pred = art_detector.predict(X + 1000) print("detector_nn_module", y_pred.shape) print("art detector ok") print("y pred ", y_pred) print("detected by detector used by attack ", np.mean(y_pred.argmax(axis=1) == 1)) clip_fun = None ################################################################# pred_folder = 'result_{:}/predictions_wb_eval/{:}_{:}_{:}'.format( seed, dataset_name, clf_name, detector_name) print("compute prediction for samples at epsilon 0") x = X_att_test[:10] y = y_att_test[:10] # compute and save predictions cmpt_and_save_predictions(art_classifier, full_magnet_orig, art_detector, device, x, y, pred_folder, 0) for eps in epsilon_lst: print("epsilon ", eps) print("detector threshold ", detector_nn_module.detector.threshold) attack = AutoProjectedGradientDescentDetectorsMagnet( estimator=art_classifier, detector=art_detector, detector_th=0, detector_clip_fun=clip_fun, loss_type='logits_difference', batch_size=128, norm=2, eps=eps, eps_step=0.9, beta=1.0, max_iter=100) adv_x = attack.generate(x=x, y=None) # compute and save predictions cmpt_and_save_predictions(art_classifier, full_magnet_orig, art_detector, device, adv_x, y, pred_folder, eps)
def main(): set_seeds(SEED) device = device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') print('device:', device) # Load classifier file_model = os.path.join('result_0', 'mnist_dnn_model.pt') model = BaseModel(use_prob=False).to(device) model.load_state_dict(torch.load(file_model, map_location=device)) file_data = os.path.join('result_0', 'mnist_dnn_apgd2_3000.pt') obj = torch.load(file_data) X = obj['X'] y = obj['y'] adv = obj['adv'] pred = predict_numpy(model, X, device) print('Acc on clean:', np.mean(pred == y)) pred = predict_numpy(model, adv, device) print('Acc on adv:', np.mean(pred == y)) # Split data X_def_test = X[:1000] y_def_test = y[:1000] adv_def_test = adv[:1000] pred_adv_def_test = pred[:1000] X_def_val = X[1000:2000] y_def_val = y[1000:2000] adv_def_val = adv[1000:2000] pred_adv_def_val = pred[1000:2000] X_att_test = X[2000:4000] y_att_test = y[2000:4000] adv_att_test = adv[2000:4000] pred_adv_att_test = pred[2000:4000] X_surro_train = X[4000:] y_surro_train = y[4000:] adv_surro_train = adv[4000:] pred_adv_surro_train = pred[4000:] # Load baard file_baard_train = os.path.join('result_0', 'mnist_dnn_baard_s1_train_data.pt') obj = torch.load(file_baard_train) X_baard_train_s1 = obj['X_s1'] X_baard_train = obj['X'] y_baard_train = obj['y'] stages = [] stages.append(ApplicabilityStage(n_classes=10, quantile=1., verbose=False)) stages.append( ReliabilityStage(n_classes=10, k=10, quantile=1., verbose=False)) stages.append( DecidabilityStage(n_classes=10, k=100, quantile=1., verbose=False)) detector = BAARDOperator(stages=stages) detector.stages[0].fit(X_baard_train_s1, y_baard_train) for stage in detector.stages[1:]: stage.fit(X_baard_train, y_baard_train) file_baard_threshold = os.path.join('result_0', 'mnist_dnn_baard_threshold.pt') thresholds = torch.load(file_baard_threshold)['thresholds'] detector.load(file_baard_threshold) file_surro = os.path.join('result_0', 'mnist_dnn_baard_surrogate.pt') surrogate = get_pretrained_surrogate(file_surro, device) # Test surrogate model X_test = np.concatenate((X_att_test[1000:], adv_att_test[1000:])) pred_test = predict_numpy(model, X_test, device) label_test = detector.detect(X_test, pred_test) acc = acc_on_adv(pred_test[1000:], y_att_test[1000:], label_test[1000:]) fpr = np.mean(label_test[:1000]) print('BAARD Acc_on_adv:', acc) print('BAARD FPR:', fpr) label_surro = predict_numpy(surrogate, X_test, device) acc = np.mean(label_surro == label_test) print('Acc on surrogate:', acc) loss = torch.nn.CrossEntropyLoss() optimizer_clf = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4) art_classifier = PyTorchClassifier(model=model, loss=loss, input_shape=(1, 28, 28), nb_classes=10, optimizer=optimizer_clf) optimizer_sur = torch.optim.SGD(surrogate.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4) art_detector = PyTorchClassifier(model=surrogate, loss=loss, input_shape=(1, 28, 28), nb_classes=2, optimizer=optimizer_sur) loss_multiplier = 1. / 36. clip_fun = BAARD_Clipper(detector) attack = AutoProjectedGradientDescentDetectors( estimator=art_classifier, detector=art_detector, detector_th=0, #fpr, clf_loss_multiplier=loss_multiplier, detector_clip_fun=clip_fun, loss_type='logits_difference', batch_size=128, norm=2, eps=8.0, eps_step=0.9, beta=0.5, max_iter=100) # X_toy = np.random.rand(128, 1, 28, 28).astype(np.float32) # pred_toy = art_classifier.predict(X_toy) # rejected_s1 = detector.stages[0].predict(X_toy, pred_toy) # print('Without:', np.mean(rejected_s1)) # X_clipped = clip_fun(X_toy, art_classifier) # rejected_s1 = detector.stages[0].predict(X_clipped, pred_toy) # print('With:', np.mean(rejected_s1)) # adv_x = attack.generate(x=X_toy) # pred_adv = predict_numpy(model, adv_x, device) # pred_sur = art_detector.predict(adv_x) # print('From surrogate model:', np.mean(pred_sur == 1)) # labelled_as_adv = detector.detect(adv_x, pred_adv) # print('From BAARD', np.mean(labelled_as_adv == 1)) # # Test it stage by stage # reject_s1 = detector.stages[0].predict(adv_x, pred_adv) # print('reject_s1', np.mean(reject_s1)) # reject_s2 = detector.stages[1].predict(adv_x, pred_adv) # print('reject_s2', np.mean(reject_s2)) # reject_s3 = detector.stages[2].predict(adv_x, pred_adv) # print('reject_s3', np.mean(reject_s3)) x = X_att_test[:10] y = y_att_test[:10] adv_x = attack.generate(x=x, y=None) pred_adv = predict_numpy(model, adv_x, device) pred_sur = art_detector.predict(adv_x) pred = predict_numpy(model, adv_x, device) print('Acc classifier:', np.mean(pred == y)) print('From surrogate model:', np.mean(pred_sur == 1)) labelled_as_adv = detector.detect(adv_x, pred_adv) print('From BAARD', np.mean(labelled_as_adv == 1)) # Test it stage by stage reject_s1 = detector.stages[0].predict(adv_x, pred_adv) print('reject_s1', np.mean(reject_s1)) reject_s2 = detector.stages[1].predict(adv_x, pred_adv) print('reject_s2', np.mean(reject_s2)) reject_s3 = detector.stages[2].predict(adv_x, pred_adv) print('reject_s3', np.mean(reject_s3)) print()
def main(): with open('data.json') as data_json: data_params = json.load(data_json) parser = argparse.ArgumentParser() parser.add_argument('--data', type=str, required=True) parser.add_argument('--model', type=str, required=True) parser.add_argument('--pretrained', type=str, required=True) parser.add_argument('--data_path', type=str, default='data') parser.add_argument('--output_path', type=str, default='results') parser.add_argument('--random_state', type=int, default=1234) args = parser.parse_args() print(args) set_seeds(args.random_state) if not os.path.exists(args.output_path): print('Output folder does not exist. Create:', args.output_path) os.mkdir(args.output_path) print('data:', args.data) print('model:', args.model) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('Device: {}'.format(device)) # Prepare data transforms = tv.transforms.Compose([tv.transforms.ToTensor()]) if args.data == 'mnist': dataset_train = datasets.MNIST(args.data_path, train=True, download=True, transform=transforms) dataset_test = datasets.MNIST(args.data_path, train=False, download=True, transform=transforms) elif args.data == 'cifar10': dataset_train = datasets.CIFAR10(args.data_path, train=True, download=True, transform=transforms) dataset_test = datasets.CIFAR10(args.data_path, train=False, download=True, transform=transforms) else: data_path = os.path.join(args.data_path, data_params['data'][args.data]['file_name']) print('Read file:', data_path) X, y = load_csv(data_path) X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=data_params['data'][args.data]['n_test'], random_state=args.random_state) scaler = MinMaxScaler().fit(X_train) X_train = scaler.transform(X_train) X_test = scaler.transform(X_test) dataset_train = TensorDataset( torch.from_numpy(X_train).type(torch.float32), torch.from_numpy(y_train).type(torch.long)) dataset_test = TensorDataset( torch.from_numpy(X_test).type(torch.float32), torch.from_numpy(y_test).type(torch.long)) loader_train = DataLoader(dataset_train, batch_size=512, shuffle=False) loader_test = DataLoader(dataset_test, batch_size=512, shuffle=False) shape_train = get_shape(loader_train.dataset) shape_test = get_shape(loader_test.dataset) print('Train set:', shape_train) print('Test set:', shape_test) use_prob = True print('Using softmax layer:', use_prob) # Load model if args.data == 'mnist': model = BaseModel(use_prob=use_prob).to(device) model_name = 'basic' elif args.data == 'cifar10': model_name = args.pretrained.split('_')[1] if model_name == 'resnet': model = Resnet(use_prob=use_prob).to(device) elif model_name == 'vgg': model = Vgg(use_prob=use_prob).to(device) else: raise NotImplementedError else: n_features = data_params['data'][args.data]['n_features'] n_classes = data_params['data'][args.data]['n_classes'] model = NumericModel(n_features, n_hidden=n_features * 4, n_classes=n_classes, use_prob=use_prob).to(device) model_name = 'basic' + str(n_features * 4) loss = nn.CrossEntropyLoss() pretrained_path = os.path.join(args.output_path, args.pretrained) model.load_state_dict(torch.load(pretrained_path, map_location=device)) _, acc_train = validate(model, loader_train, loss, device) _, acc_test = validate(model, loader_test, loss, device) print('Accuracy on train set: {:.4f}%'.format(acc_train * 100)) print('Accuracy on test set: {:.4f}%'.format(acc_test * 100)) # Create a subset which only contains recognisable samples. # The original train and test sets are no longer needed. tensor_train_X, tensor_train_y = get_correct_examples(model, dataset_train, device=device, return_tensor=True) dataset_train = TensorDataset(tensor_train_X, tensor_train_y) loader_train = DataLoader(dataset_train, batch_size=512, shuffle=False) _, acc_perfect = validate(model, loader_train, loss, device) print('Accuracy on {} filtered train set: {:.4f}%'.format( len(dataset_train), acc_perfect * 100)) tensor_test_X, tensor_test_y = get_correct_examples(model, dataset_test, device=device, return_tensor=True) dataset_test = TensorDataset(tensor_test_X, tensor_test_y) loader_test = DataLoader(dataset_test, batch_size=512, shuffle=False) _, acc_perfect = validate(model, loader_test, loss, device) print('Accuracy on {} filtered test set: {:.4f}%'.format( len(dataset_test), acc_perfect * 100)) X_train = tensor_train_X.cpu().detach().numpy() y_train = tensor_train_y.cpu().detach().numpy() X_baard = baard_preprocess(args.data, tensor_train_X).cpu().detach().numpy() obj = {'X_train': X_baard, 'y_train': y_train} path_ouput = os.path.join( args.output_path, '{}_{}_baard_train.pt'.format(args.data, args.model, args.model)) torch.save(obj, path_ouput) print('Save to:', path_ouput) print()
def main(): parser = argparse.ArgumentParser() parser.add_argument('--data', type=str, required=True, choices=DATA_NAMES) parser.add_argument('--data_path', type=str, default='data') parser.add_argument('--output_path', type=str, default='results') parser.add_argument('--pretrained', type=str, required=True) parser.add_argument('--param', type=str, required=True) parser.add_argument('--random_state', type=int, default=1234) args = parser.parse_args() print(args) set_seeds(args.random_state) if not os.path.exists(args.output_path): print('Output folder does not exist. Create:', args.output_path) os.mkdir(args.output_path) with open(args.param) as param_json: param = json.load(param_json) param['n_classes'] = DATA[args.data]['n_classes'] device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('Dataset:', args.data) print('Pretrained model:', args.pretrained) print('Param:', param) print('Device: {}'.format(device)) transforms = tv.transforms.Compose([tv.transforms.ToTensor()]) # the autoencoder2 need a larger temperature value for the softmax function. use_prob = False if args.data == 'mnist': dataset_train = datasets.MNIST( args.data_path, train=True, download=True, transform=transforms) dataset_test = datasets.MNIST( args.data_path, train=False, download=True, transform=transforms) model = BaseModel(use_prob=use_prob).to(device) model_name = 'basic' elif args.data == 'cifar10': dataset_train = datasets.CIFAR10( args.data_path, train=True, download=True, transform=transforms) dataset_test = datasets.CIFAR10( args.data_path, train=False, download=True, transform=transforms) model_name = args.pretrained.split('_')[1] if model_name == 'resnet': model = Resnet(use_prob=use_prob).to(device) elif model_name == 'vgg': model = Vgg(use_prob=use_prob).to(device) else: raise ValueError('model_name must be either resnet or vgg.') else: raise ValueError('This autoencoder does not support other datasets.') tensor_X_train, tensor_y_train = dataset2tensor(dataset_train) X_train = tensor_X_train.cpu().detach().numpy() y_train = tensor_y_train.cpu().detach().numpy() X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=5000) loader_train = DataLoader( dataset_train, batch_size=512, shuffle=False) loader_test = DataLoader(dataset_test, batch_size=512, shuffle=False) loss = nn.CrossEntropyLoss() pretrained_path = os.path.join(args.output_path, args.pretrained) model.load_state_dict(torch.load(pretrained_path)) _, acc_train = validate(model, loader_train, loss, device) _, acc_test = validate(model, loader_test, loss, device) print('Accuracy on train set: {:.4f}%'.format(acc_train*100)) print('Accuracy on test set: {:.4f}%'.format(acc_test*100)) if args.data == 'mnist': detector1 = MagNetDetector( encoder=Autoencoder1(n_channel=DATA[args.data]['n_features'][0]), classifier=model, lr=param['lr'], batch_size=param['batch_size'], weight_decay=param['weight_decay'], x_min=0.0, x_max=1.0, noise_strength=param['noise_strength'], algorithm='error', p=1, device=device) detector1.fit(X_train, y_train, epochs=param['epochs']) detector2 = MagNetDetector( encoder=Autoencoder2(n_channel=DATA[args.data]['n_features'][0]), classifier=model, lr=param['lr'], batch_size=param['batch_size'], weight_decay=param['weight_decay'], x_min=0.0, x_max=1.0, noise_strength=param['noise_strength'], algorithm='error', p=2, device=device) detector2.fit(X_train, y_train, epochs=param['epochs']) detectors = [detector1, detector2] elif args.data == 'cifar10': autoencoder = Autoencoder2(n_channel=DATA[args.data]['n_features'][0]) detectors = [] detector = MagNetDetector( encoder=autoencoder, classifier=model, lr=param['lr'], batch_size=param['batch_size'], weight_decay=param['weight_decay'], x_min=0.0, x_max=1.0, noise_strength=param['noise_strength'], algorithm='error', p=2, device=device) detector.fit(X_train, y_train, epochs=param['epochs']) detectors.append(detector) detectors.append(MagNetDetector( encoder=autoencoder, classifier=model, lr=param['lr'], batch_size=param['batch_size'], weight_decay=param['weight_decay'], x_min=0.0, x_max=1.0, noise_strength=param['noise_strength'], algorithm='prob', temperature=10, device=device)) detectors.append(MagNetDetector( encoder=autoencoder, classifier=model, lr=param['lr'], batch_size=param['batch_size'], weight_decay=param['weight_decay'], x_min=0.0, x_max=1.0, noise_strength=param['noise_strength'], algorithm='prob', temperature=40, device=device)) else: raise ValueError('Unsupported dataset.') # Train autoencoders for ae in detectors: mse = ae.score(X_val) print('MSE training set: {:.6f}, validation set: {:.6f}'.format( ae.history_train_loss[-1] if len(ae.history_train_loss) > 0 else np.inf, mse)) ae.search_threshold(X_val, fp=param['fp'], update=True) print('Threshold:', ae.threshold) # Save autoencoders for i, ae in enumerate(detectors, start=1): encoder_path = os.path.join( args.output_path, 'autoencoder_{}_{}_{}.pt'.format(args.data, model_name, i)) ae.save(encoder_path) print('File is saved to:', encoder_path)
def main(): with open('data.json') as data_json: data_params = json.load(data_json) parser = argparse.ArgumentParser() parser.add_argument('--data', type=str, required=True) parser.add_argument('--data_path', type=str, default='data') parser.add_argument('--output_path', type=str, default='results') parser.add_argument('--batch_size', type=int, default=128) parser.add_argument('--epochs', type=int, default=5) parser.add_argument('--random_state', type=int, default=1234) args = parser.parse_args() print(args) set_seeds(args.random_state) if not os.path.exists(args.output_path): print('Output folder does not exist. Create:', args.output_path) os.mkdir(args.output_path) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('Device: {}'.format(device)) # Prepare data data_path = os.path.join(args.data_path, data_params['data'][args.data]['file_name']) print('Read file: {}'.format(data_path)) X, y = load_csv(data_path) # Normalize data scaler = MinMaxScaler().fit(X) X = scaler.transform(X) n_test = data_params['data'][args.data]['n_test'] random_state = args.random_state X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=n_test, random_state=random_state) dataset_train = TensorDataset( torch.from_numpy(X_train).type(torch.float32), torch.from_numpy(y_train).type(torch.long)) dataset_test = TensorDataset( torch.from_numpy(X_test).type(torch.float32), torch.from_numpy(y_test).type(torch.long)) dataloader_train = DataLoader(dataset_train, args.batch_size, shuffle=True) dataloader_test = DataLoader(dataset_test, args.batch_size, shuffle=False) print('Train set: {}, Test set: {}'.format(X_train.shape, X_test.shape)) # Prepare model n_features = data_params['data'][args.data]['n_features'] n_classes = data_params['data'][args.data]['n_classes'] print('n_features: {}, n_classes: {}'.format(n_features, n_classes)) model = NumericModel(n_features, n_hidden=n_features * 4, n_classes=n_classes).to(device) optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4) loss = nn.CrossEntropyLoss() scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=args.epochs, eta_min=1e-7) # Train model since = time.time() for epoch in range(args.epochs): start = time.time() tr_loss, tr_acc = train(model, dataloader_train, loss, optimizer, device) va_loss, va_acc = validate(model, dataloader_test, loss, device) scheduler.step() time_elapsed = time.time() - start if epoch % 10 == 0: print( '{:2d}/{:d}[{:s}] Train Loss: {:.4f} Acc: {:.4f}%, Test Loss: {:.4f} Acc: {:.4f}%' .format(epoch + 1, args.epochs, str(datetime.timedelta(seconds=time_elapsed)), tr_loss, tr_acc * 100, va_loss, va_acc * 100)) time_elapsed = time.time() - since print('Total run time: {:.0f}m {:.1f}s'.format(time_elapsed // 60, time_elapsed % 60)) # Save model file_name = os.path.join(args.output_path, '{}_{}.pt'.format(args.data, args.epochs)) print('Output file name: {}'.format(file_name)) torch.save(model.state_dict(), file_name) # Test accuracy per class: print('Training set:') X, y = dataset2tensor(dataset_train) X = X.cpu().detach().numpy() y = y.cpu().detach().numpy() print_acc_per_label(model, X, y, device) print('Test set:') X, y = dataset2tensor(dataset_test) X = X.cpu().detach().numpy() y = y.cpu().detach().numpy() print_acc_per_label(model, X, y, device)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--data_path', type=str, default='data') parser.add_argument('--output_path', type=str, default='results') parser.add_argument('--batch_size', type=int, default=128) parser.add_argument('--epochs', type=int, default=5) parser.add_argument('--pretrained', type=str, nargs='?') parser.add_argument('--random_state', type=int, default=1234) args = parser.parse_args() print(args) set_seeds(args.random_state) if not os.path.exists(args.data_path): os.makedirs(args.data_path) if not os.path.exists(args.output_path): os.makedirs(args.output_path) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('Device: {}'.format(device)) # Fetch dataset transforms = tv.transforms.Compose([tv.transforms.ToTensor()]) dataset_train = datasets.MNIST( args.data_path, train=True, download=True, transform=transforms) dataset_test = datasets.MNIST( args.data_path, train=False, download=True, transform=transforms) dataloader_train = DataLoader( dataset_train, batch_size=args.batch_size, shuffle=True) dataloader_test = DataLoader( dataset_test, batch_size=args.batch_size, shuffle=False) print('Train set: {}, Test set: {}'.format( len(dataset_train), len(dataset_test))) # Prepare model model = BaseModel().to(device) optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4) loss = nn.CrossEntropyLoss() scheduler = optim.lr_scheduler.CosineAnnealingLR( optimizer, T_max=args.epochs) # Load pre-trained model if args.pretrained is not None: pretrained_path = os.path.join(args.output_path, args.pretrained) model.load_state_dict(torch.load(pretrained_path, map_location=device)) # Train model since = time.time() for epoch in range(args.epochs): start = time.time() tr_loss, tr_acc = train(model, dataloader_train, loss, optimizer, device) va_loss, va_acc = validate(model, dataloader_test, loss, device) scheduler.step() time_elapsed = time.time() - start print(('{:2d}/{:d}[{:s}] Train Loss: {:.4f} Acc: {:.4f}%, ' + 'Test Loss: {:.4f} Acc: {:.4f}%').format( epoch + 1, args.epochs, str(datetime.timedelta(seconds=time_elapsed)), tr_loss, tr_acc * 100., va_loss, va_acc * 100.)) time_elapsed = time.time() - since print('Total run time: {:.0f}m {:.1f}s'.format( time_elapsed // 60, time_elapsed % 60)) # Save model file_name = os.path.join( args.output_path, 'mnist_{}.pt'.format(args.epochs)) print('Output file name: {}'.format(file_name)) torch.save(model.state_dict(), file_name) # Test accuracy per class: print('Training set:') X, y = dataset2tensor(dataset_train) X = X.cpu().detach().numpy() y = y.cpu().detach().numpy() print_acc_per_label(model, X, y, device) print('Test set:') X, y = dataset2tensor(dataset_test) X = X.cpu().detach().numpy() y = y.cpu().detach().numpy() print_acc_per_label(model, X, y, device)
def main(): with open('data.json') as data_json: data_params = json.load(data_json) parser = argparse.ArgumentParser() parser.add_argument('--data', type=str, required=True, choices=data_params['datasets']) parser.add_argument('--model', type=str, required=True, choices=['svm', 'tree']) parser.add_argument('--adv', type=str, required=True, help="Example: 'apgd_0.3'") parser.add_argument('--defence', type=str, required=True, choices=['baard', 'rc']) parser.add_argument('--param', type=str, required=True) parser.add_argument('--suffix', type=str, default=None) parser.add_argument('--data_path', type=str, default='data') parser.add_argument('--output_path', type=str, default='results') parser.add_argument('--save', type=int, default=1, choices=[0, 1]) parser.add_argument('--random_state', type=int, default=1234) args = parser.parse_args() args.adv = '{}_{}_{}'.format(args.data, args.model, args.adv) print(args) set_seeds(args.random_state) if not os.path.exists(args.output_path): print('Output folder does not exist. Create:', args.output_path) os.mkdir(args.output_path) print('Dataset:', args.data) print('Model:', args.model) print('Pretrained samples:', args.adv + '_adv.npy') print('Defence:', args.defence) with open(args.param) as param_json: param = json.load(param_json) param['n_classes'] = data_params['data'][args.data]['n_classes'] print('Param:', param) # Prepare data data_path = os.path.join(args.data_path, data_params['data'][args.data]['file_name']) print('Read file: {}'.format(data_path)) X, y = load_csv(data_path) # Apply scaling scaler = MinMaxScaler().fit(X) X = scaler.transform(X) n_test = data_params['data'][args.data]['n_test'] random_state = args.random_state X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=n_test, random_state=random_state) # Train model if args.model == 'svm': model = SVC(kernel="linear", C=1.0, gamma="scale", random_state=random_state) elif args.model == 'tree': model = ExtraTreeClassifier(random_state=random_state) else: raise NotImplementedError model.fit(X_train, y_train) acc_train = model.score(X_train, y_train) acc_test = model.score(X_test, y_test) print(('Train Acc: {:.4f}, ' + 'Test Acc: {:.4f}').format(acc_train, acc_test)) # Load adversarial examples path_x = os.path.join(args.output_path, '{}_x.npy'.format(args.adv)) path_y = os.path.join(args.output_path, '{}_y.npy'.format(args.adv)) path_adv = os.path.join(args.output_path, '{}_adv.npy'.format(args.adv)) X_benign = np.load(path_x) y_true = np.load(path_y) adv = np.load(path_adv) print('Acc on clean:', model.score(X_benign, y_true)) print('Acc on adv:', model.score(adv, y_true)) n = len(X_benign) // 2 X_val = X_benign[n:] y_val = y_true[n:] adv_val = adv[n:] X_test = X_benign[:n] y_test = y_true[:n] adv_test = adv[:n] # Train defence time_start = time.time() if args.defence == 'baard': sequence = param['sequence'] stages = [] if sequence[0]: stages.append(ApplicabilityStage(n_classes=param['n_classes'], quantile=param['q1'])) if sequence[1]: stages.append(ReliabilityStage(n_classes=param['n_classes'], k=param['k_re'], quantile=param['q2'])) if sequence[2]: stages.append(DecidabilityStage(n_classes=param['n_classes'], k=param['k_de'], quantile=param['q3'])) print('BAARD: # of stages:', len(stages)) detector = BAARDOperator(stages=stages) # Run preprocessing baard_train_path = os.path.join('results', '{}_{}_baard_train.pt'.format(args.data, args.model)) obj = torch.load(baard_train_path) X_baard = obj['X_train'] y_train = obj['y_train'] detector.stages[0].fit(X_baard, y_train) detector.stages[1].fit(X_train, y_train) if len(detector.stages) == 3: detector.stages[2].fit(X_train, y_train) detector.search_thresholds(X_val, model.predict(X_val), np.zeros_like(y_val)) path_baard = os.path.join(args.output_path, 'baard_{}_{}_param.pt'.format(args.data, args.model)) detector.save(path_baard) elif args.defence == 'rc': detector = SklearnRegionBasedClassifier( model=model, r=param['r'], sample_size=1000, n_classes=param['n_classes'], x_min=0.0, x_max=1.0, r0=0.0, step_size=0.02, stop_value=0.4) else: raise NotImplementedError time_elapsed = time.time() - time_start print('Total training time:', str(datetime.timedelta(seconds=time_elapsed))) # Test defence time_start = time.time() X_test, labels_test = merge_and_generate_labels(adv_test, X_test, flatten=False) pred_adv = model.predict(adv_test) pred_test = np.concatenate((pred_adv, y_test)) y_test = np.concatenate((y_test, y_test)) if args.defence == 'baard': res_test = detector.detect(X_test, pred_test) elif args.defence == 'rc': pred_test = detector.detect(X_test, pred_test) res_test = np.zeros_like(pred_test) else: raise NotImplementedError acc = acc_on_adv(pred_test[:n], y_test[:n], res_test[:n]) if args.defence == 'rc': fpr = np.mean(pred_test[n:] != y_test[n:]) else: fpr = np.mean(res_test[n:]) print('Acc_on_adv:', acc) print('FPR:', fpr) time_elapsed = time.time() - time_start print('Total test time:', str(datetime.timedelta(seconds=time_elapsed))) # Save results suffix = '_' + args.suffix if args.suffix is not None else '' obj = { 'X_test': X_test, 'y_test': y_test, 'labels_test': labels_test, 'res_test': pred_test if args.defence == 'rc' else res_test, 'param': param} if args.save: path_result = os.path.join(args.output_path, '{}_{}{}.pt'.format(args.adv, args.defence, suffix)) torch.save(obj, path_result) print('Saved to:', path_result) else: print('No file is save!') print()
def main(): set_seeds(SEED) device = device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') print('device:', device) # Load classifier file_model = os.path.join('result_0', 'mnist_dnn_model.pt') model = BaseModel(use_prob=False).to(device) model.load_state_dict(torch.load(file_model, map_location=device)) file_data = os.path.join('result_0', 'mnist_dnn_apgd2_3000.pt') obj = torch.load(file_data) X = obj['X'] y = obj['y'] adv = obj['adv'] pred = predict_numpy(model, X, device) print('Acc on clean:', np.mean(pred == y)) pred = predict_numpy(model, adv, device) print('Acc on adv:', np.mean(pred == y)) # Split data X_def_test = X[:1000] y_def_test = y[:1000] adv_def_test = adv[:1000] pred_adv_def_test = pred[:1000] X_def_val = X[1000:2000] y_def_val = y[1000:2000] adv_def_val = adv[1000:2000] pred_adv_def_val = pred[1000:2000] X_att_test = X[2000:4000] y_att_test = y[2000:4000] adv_att_test = adv[2000:4000] pred_adv_att_test = pred[2000:4000] X_surro_train = X[4000:] y_surro_train = y[4000:] adv_surro_train = adv[4000:] pred_adv_surro_train = pred[4000:] # Load baard file_baard_train = os.path.join( 'result_0', 'mnist_dnn_baard_s1_train_data.pt') obj = torch.load(file_baard_train) X_baard_train_s1 = obj['X_s1'] X_baard_train = obj['X'] y_baard_train = obj['y'] file_baard_threshold = os.path.join( 'result_0', 'mnist_dnn_baard_threshold.pt') thresholds = torch.load(file_baard_threshold)['thresholds'] stage1 = ApplicabilityStage(n_classes=10, quantile=1.) stage1.thresholds_ = thresholds[0] file_surro = os.path.join('result_0', 'mnist_dnn_baard_surrogate.pt') surrogate = get_pretrained_surrogate(file_surro, device) # Test surrogate model X_test = np.concatenate((X_att_test[1000:], adv_att_test[1000:])) pred_test = predict_numpy(model, X_test, device) # label_test = detector.detect(X_test, pred_test) # acc = acc_on_adv(pred_test[1000:], y_att_test[1000:], label_test[1000:]) # fpr = np.mean(label_test[:1000]) # print('BAARD Acc_on_adv:', acc) # print('BAARD FPR:', fpr) label_surro = predict_numpy(surrogate, X_test, device) # acc = np.mean(label_surro == label_test) # print('Acc on surrogate:', acc) loss = torch.nn.CrossEntropyLoss() optimizer_clf = torch.optim.SGD( model.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4) art_classifier = PyTorchClassifier( model=model, loss=loss, input_shape=(1, 28, 28), nb_classes=10, optimizer=optimizer_clf ) optimizer_sur = torch.optim.SGD( surrogate.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4) art_detector = PyTorchClassifier( model=surrogate, loss=loss, input_shape=(1, 28, 28), nb_classes=2, optimizer=optimizer_sur ) fpr = 0.05 attack = AutoProjectedGradientDescentDetectors( estimator=art_classifier, detector=art_detector, detector_th=fpr, clf_loss_multiplier=1. / 36., loss_type='logits_difference', batch_size=128, norm=2, eps=5.0, eps_step=0.9, beta=0.5, max_iter=100) # adv_x = attack.generate(x=X_att_test[:100], y=y_att_test[:100]) file_whitebox_adv = 'mnist_apgd2_3000_whitebox_size100.npy' # np.save(file_whitebox_adv, adv_x) adv_x = np.load(file_whitebox_adv) print('adv_x', adv_x.shape) pred_adv = predict_numpy(model, adv_x, device) adv_x = clip_by_threshold(adv_x, pred_adv, thresholds[0]) pred_sur = art_detector.predict(adv_x) print('From surrogate model:', np.mean(pred_sur == 1)) labelled_as_adv = stage1.predict(adv_x, pred_adv) print('From BAARD', np.mean(labelled_as_adv == 1)) # Testing # X_toy = np.random.rand(128, 1, 28, 28).astype(np.float32) # Same size as MNIST in a single batch # y_toy = np.concatenate((np.zeros(50), np.ones(50))) # rejected = stage1.predict(X_toy, y_toy) # print('rejected', np.mean(rejected)) # X_bypass = clip_by_threshold(X_toy, y_toy, thresholds[0]) # rejected_after = stage1.predict(X_bypass, y_toy) # print('rejected_after', np.mean(rejected_after)) print('Pause')