## test_defense.py -- test defense
##
## Copyright (C) 2017, Dongyu Meng <*****@*****.**>.
##
## This program is licenced under the BSD 2-Clause licence,
## contained in the LICENCE file in this directory.
from setup_mnist import MNIST
from mn_utils import prepare_data
from worker import AEDetector, SimpleReformer, IdReformer, AttackData, Classifier, Operator, Evaluator
import mn_utils as utils
detector_I = AEDetector("./defensive_models/MNIST_I", p=2)
detector_II = AEDetector("./defensive_models/MNIST_II", p=1)
reformer = SimpleReformer("./defensive_models/MNIST_I")
id_reformer = IdReformer()
classifier = Classifier("./models/example_classifier")
detector_dict = dict()
detector_dict["I"] = detector_I
detector_dict["II"] = detector_II
operator = Operator(MNIST(), classifier, detector_dict, reformer)
idx = utils.load_obj("example_idx")
_, _, Y = prepare_data(MNIST(), idx)
f = "example_carlini_0.0"
testAttack = AttackData(f, Y, "Carlini L2 0.0")
def test(model, args):
data_path = args.data_path
n_channels = args.channels
n_classes = args.classes
data_width = args.width
data_height = args.height
gpu = args.gpu
# Hyper paremter for MagNet
thresholds = [0.01, 0.05, 0.001, 0.005]
reformer_model = None
if args.reformer == 'autoencoder1':
reformer_model = autoencoder(n_channels)
elif args.reformer == 'autoencoder2':
reformer_model = autoencoder2(n_channels)
else:
print("wrong reformer model : must be autoencoder1 or autoencoder2")
raise SystemExit
print('reformer model')
summary(reformer_model,
input_size=(n_channels, data_height, data_width),
device='cpu')
detector_model = None
if args.detector == 'autoencoder1':
detector_model = autoencoder(n_channels)
elif args.detector == 'autoencoder2':
detector_model = autoencoder2(n_channels)
else:
print("wrong detector model : must be autoencoder1 or autoencoder2")
raise SystemExit
print('detector model')
summary(detector_model,
input_size=(n_channels, data_height, data_width),
device='cpu')
# set device configuration
device_ids = []
if gpu == 'gpu':
if not torch.cuda.is_available():
print("No cuda available")
raise SystemExit
device = torch.device(args.model_device1)
device_defense = torch.device(args.defense_model_device)
device_ids.append(args.model_device1)
if args.model_device2 != -1:
device_ids.append(args.model_device2)
if args.model_device3 != -1:
device_ids.append(args.model_device3)
if args.model_device4 != -1:
device_ids.append(args.model_device4)
else:
device = torch.device("cpu")
device_defense = torch.device("cpu")
detector = AEDetector(detector_model,
device_defense,
args.detector_path,
p=2)
reformer = SimpleReformer(reformer_model, device_defense,
args.reformer_path)
classifier = Classifier(model, device, args.model_path, device_ids)
# set testdataset
test_dataset = SampleDataset(data_path)
test_loader = DataLoader(
test_dataset,
batch_size=10,
num_workers=4,
)
print('test_dataset : {}, test_loader : {}'.format(len(test_dataset),
len(test_loader)))
# Defense with MagNet
print('test start')
for thrs in thresholds:
print('----------------------------------------')
counter = 0
avg_score = 0.0
thrs = torch.tensor(thrs)
with torch.no_grad():
for batch_idx, (inputs, labels) in enumerate(test_loader):
inputs = inputs.float()
labels = labels.to(device).long()
target = make_one_hot(labels[:, 0, :, :], n_classes, device)
operate_results = operate(reformer, classifier, inputs)
all_pass, _ = filters(detector, inputs, thrs)
if len(all_pass) == 0:
continue
filtered_results = operate_results[all_pass]
pred = filtered_results.to(device).float()
target = target[all_pass]
loss = dice_score(pred, target)
avg_score += loss.data.cpu().numpy()
# statistics
counter += 1
del inputs, labels, pred, target, loss
if counter:
avg_score = avg_score / counter
print('threshold : {:.4f}, avg_score : {:.4f}'.format(
thrs, avg_score))
else:
print(
'threshold : {:.4f} , no images pass from filter'.format(thrs))
## test_defense.py -- test defense
##
## Copyright (C) 2017, Dongyu Meng <*****@*****.**>.
##
## This program is licenced under the BSD 2-Clause licence,
## contained in the LICENCE file in this directory.
from setup_mnist import MNIST
from utils import prepare_data
from worker import AEDetector, SimpleReformer, IdReformer, AttackData, Classifier, Operator, Evaluator
import utils
detector_I = AEDetector("./defensive_models/FACE_I", p=2)
detector_II = AEDetector("./defensive_models/FACE_II", p=1)
reformer = SimpleReformer("./defensive_models/FACE_I")
id_reformer = IdReformer()
classifier = Classifier("./models/example_classifier")
detector_dict = dict()
detector_dict["I"] = detector_I
detector_dict["II"] = detector_II
operator = Operator(MNIST(), classifier, detector_dict, reformer)
idx = utils.load_obj("example_idx")
_, _, Y = prepare_data(MNIST(), idx)
f = "example_carlini_0.0"
testAttack = AttackData(f, Y, "Carlini L2 0.0")
evaluator = Evaluator(operator, testAttack)
p=2) for i in range(8)
]
if attacking:
detector_II += [
AEDetector("./defensive_models/MNIST_II_" + str(i), p=2)
for i in range(8)
]
else:
detector_II = [
AEDetector("./defensive_models/" + prefix + "PAE_CIFAR_II_" + str(i),
p=1) for i in range(8)
]
if dataset == "MNIST":
reformer = [
SimpleReformer("./defensive_models/" + prefix + "PAE_MNIST_I_" +
str(i)) for i in range(8)
]
if attacking:
reformer += [
SimpleReformer("./defensive_models/MNIST_I_" + str(i))
for i in range(24)
]
else:
reformer = [
SimpleReformer("./defensive_models/" + prefix + "PAE_CIFAR_II_" +
str(i)) for i in range(8)
]
id_reformer = IdReformer()
if dataset == "MNIST":