def evaluate(self):
"""
Evaluate.
"""
self.model.eval()
clean_probabilities = common.test.test(self.model,
self.testloader,
cuda=self.args.cuda)
attack, objective = self.get_attack()
adversarial_perturbations, adversarial_probabilities, adversarial_errors = common.test.attack(
self.model,
self.adversarialloader,
attack,
objective,
attempts=5,
cuda=self.args.cuda)
eval = common.eval.AdversarialEvaluation(clean_probabilities,
adversarial_probabilities,
self.testset.labels,
validation=0.9,
errors=adversarial_errors)
log('test error in %%: %g' % (eval.test_error() * 100))
log('test error @99%%tpr in %%: %g' %
(eval.test_error_at_99tpr() * 100))
log('robust test error in %%: %g' % (eval.robust_test_error() * 100))
log('robust test error @99%%tpr in %%: %g' %
(eval.robust_test_error_at_99tpr() * 100))
def testTrainModel(self, model):
if self.cuda:
model = model.cuda()
optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
scheduler = common.train.get_exponential_scheduler(
optimizer, batches_per_epoch=len(self.trainset))
writer = common.summary.SummaryDictWriter()
augmentation = None
trainer = common.train.NormalTraining(model,
self.trainset,
self.testset,
optimizer,
scheduler,
augmentation=augmentation,
writer=writer,
cuda=self.cuda)
trainer.summary_gradients = False
epochs = 20
for e in range(epochs):
trainer.step(e)
probabilities = common.test.test(model, self.testset, cuda=self.cuda)
eval = common.eval.CleanEvaluation(probabilities,
self.testset.dataset.labels,
validation=0)
print(eval.test_error())
return eval
def testNormalTrainingAugmentation(self):
optimizer = torch.optim.SGD(self.model.parameters(),
lr=0.1,
momentum=0.9)
scheduler = common.train.get_exponential_scheduler(
optimizer, batches_per_epoch=len(self.trainset))
writer = common.summary.SummaryDictWriter()
augmentation = iaa.Sequential([
iaa.AdditiveGaussianNoise(scale=(0, 0.025)),
iaa.Add((-0.075, 0.075)),
common.imgaug.Clip(0, 1)
])
trainer = common.train.NormalTraining(self.model,
self.trainset,
self.testset,
optimizer,
scheduler,
augmentation=augmentation,
writer=writer,
cuda=self.cuda)
trainer.summary_gradients = False
epochs = 10
for e in range(epochs):
trainer.step(e)
probabilities = common.test.test(self.model,
self.testset,
cuda=self.cuda)
eval = common.eval.CleanEvaluation(probabilities,
self.testset.dataset.labels,
validation=0)
self.assertGreaterEqual(0.03, eval.test_error())
def setUpClass(cls):
cls.batch_size = 100
cls.cuda = True
cls.setDatasets()
if os.path.exists(cls.getModelFile()):
state = common.state.State.load(cls.getModelFile())
cls.model = state.model
if cls.cuda:
cls.model = cls.model.cuda()
else:
cls.model = cls.getModel()
if cls.cuda:
cls.model = cls.model.cuda()
optimizer = torch.optim.SGD(cls.model.parameters(),
lr=0.1,
momentum=0.9)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 5, 0.1)
writer = common.summary.SummaryWriter()
augmentation = None
trainer = common.train.NormalTraining(cls.model,
cls.trainloader,
cls.testloader,
optimizer,
scheduler,
augmentation=augmentation,
writer=writer,
cuda=cls.cuda)
for e in range(10):
trainer.step(e)
print(e)
common.state.State.checkpoint(cls.getModelFile(), cls.model,
optimizer, scheduler, e)
cls.model.eval()
probabilities = common.test.test(cls.model,
cls.testloader,
cuda=cls.cuda)
eval = common.eval.CleanEvaluation(probabilities,
cls.testloader.dataset.labels,
validation=0)
assert 0.05 > eval.test_error(), '0.05 !> %g' % eval.test_error()
assert numpy.mean(numpy.max(probabilities, axis=1)) > 0.9
def testReferenceAdversarialTraining(self):
optimizer = torch.optim.SGD(self.model.parameters(),
lr=0.1,
momentum=0.9)
scheduler = common.train.get_exponential_scheduler(
optimizer, batches_per_epoch=len(self.trainset))
writer = common.summary.SummaryDictWriter()
augmentation = None
epsilon = 0.3
attack = attacks.BatchReferencePGD()
attack.max_iterations = 10
attack.epsilon = epsilon
attack.base_lr = 0.1
attack.norm = attacks.norms.LInfNorm()
objective = attacks.objectives.UntargetedF0Objective()
trainer = common.train.AdversarialTraining(self.model,
self.trainset,
self.testset,
optimizer,
scheduler,
attack,
objective,
fraction=1,
augmentation=augmentation,
writer=writer,
cuda=self.cuda)
trainer.summary_gradients = False
epochs = 10
for e in range(epochs):
trainer.step(e)
probabilities = common.test.test(self.model,
self.testset,
cuda=self.cuda)
eval = common.eval.CleanEvaluation(probabilities,
self.testset.dataset.labels,
validation=0)
self.assertGreaterEqual(0.05, eval.test_error())
adversarial_perturbations, adversarial_probabilities, _ = common.test.attack(
self.model,
self.adversarialset,
attack,
objective,
attempts=1,
writer=writer,
cuda=self.cuda)
eval = common.eval.AdversarialEvaluation(
probabilities[:len(self.adversarialset.dataset)],
adversarial_probabilities,
self.adversarialset.dataset.labels,
validation=0)
self.assertGreaterEqual(0.2, eval.robust_test_error())
def setUpClass(cls):
cls.batch_size = 100
cls.cuda = True
cls.setDatasets()
if os.path.exists(cls.getModelFile()):
state = common.state.State.load(cls.getModelFile())
cls.model = state.model
if cls.cuda:
cls.model = cls.model.cuda()
else:
cls.model = cls.getModel()
if cls.cuda:
cls.model = cls.model.cuda()
print(cls.model)
optimizer = torch.optim.SGD(cls.model.parameters(), lr=0.1, momentum=0.9)
scheduler = common.train.get_exponential_scheduler(optimizer, batches_per_epoch=len(cls.trainloader))
writer = common.summary.SummaryWriter()
augmentation = None
attack = attacks.batch_reference_pgd.BatchReferencePGD()
attack.max_iterations = 5
attack.epsilon = 0.3
attack.base_lr = 0.1
attack.norm = attacks.norms.LInfNorm()
objective = attacks.objectives.UntargetedF0Objective()
trainer = common.train.AdversarialTraining(cls.model, cls.trainloader, cls.testloader, optimizer, scheduler, attack=attack, objective=objective, augmentation=augmentation, writer=writer, cuda=cls.cuda)
for e in range(10):
trainer.step(e)
common.state.State.checkpoint(cls.getModelFile(), cls.model, optimizer, scheduler, e)
cls.model.eval()
probabilities = common.test.test(cls.model, cls.testloader, cuda=cls.cuda)
eval = common.eval.CleanEvaluation(probabilities, cls.testloader.dataset.labels, validation=0)
assert 0.075 > eval.test_error(), '0.05 !> %g' % eval.test_error()
assert numpy.mean(numpy.max(probabilities, axis=1)) > 0.9
cls.model.eval()
def evaluate(self):
"""
Evaluate.
"""
# это не работает и это нормально. так и должно быть. для него надо отдельно код фиксить
self.model.eval()
clean_probabilities = common.test.test(self.model,
self.testloader,
cuda=True)
total_adversarial_probabilities = None
total_adversarial_errors = None
for attack, objective, attempts in self.get_test_attacks():
_, adversarial_probabilities, _ = common.test.attack(
self.model,
self.adversarialloader,
attack,
objective,
attempts=attempts,
cuda=True)
relevant_adversarial_probabilities = numpy.copy(
adversarial_probabilities)
relevant_adversarial_probabilities[:,
numpy.arange(
relevant_adversarial_probabilities
.shape[1]), self.testset.
labels[:relevant_adversarial_probabilities
.shape[1]], ] = 0
assert len(relevant_adversarial_probabilities.shape) == 3
adversarial_errors = -numpy.max(relevant_adversarial_probabilities,
axis=2)
total_adversarial_probabilities = common.numpy.concatenate(
total_adversarial_probabilities,
adversarial_probabilities,
axis=0)
total_adversarial_errors = common.numpy.concatenate(
total_adversarial_errors, adversarial_errors, axis=0)
eval = common.eval.AdversarialEvaluation(
clean_probabilities,
total_adversarial_probabilities,
self.testset.labels,
validation=0.9,
errors=total_adversarial_errors)
log('test error in %%: %g' % (eval.test_error() * 100))
log('test error @99%%tpr in %%: %g' %
(eval.test_error_at_99tpr() * 100))
log('robust test error in %%: %g' % (eval.robust_test_error() * 100))
log('robust test error @99%%tpr in %%: %g' %
(eval.robust_test_error_at_99tpr() * 100))
log(f'success rate: {eval.success_rate()}')
def testTest(self):
model = models.LeNet(10, [1, 28, 28], channels=12)
if self.cuda:
model = model.cuda()
model.eval()
probabilities = common.test.test(model, self.testset, cuda=self.cuda)
eval = common.eval.CleanEvaluation(probabilities,
self.testset.dataset.labels)
self.assertGreaterEqual(0.05, abs(0.9 - eval.test_error()))
def evaluate(self):
"""
Evaluate.
"""
self.model.eval()
clean_probabilities = common.test.test(self.model,
self.testloader,
cuda=self.args.cuda)
total_adversarial_probabilities = None
total_adversarial_errors = None
for attack, objective, attempts in self.get_test_attacks():
_, adversarial_probabilities, _ = common.test.attack(
self.model,
self.adversarialloader,
attack,
objective,
attempts=attempts,
cuda=self.args.cuda)
relevant_adversarial_probabilities = numpy.copy(
adversarial_probabilities)
relevant_adversarial_probabilities[:,
numpy.arange(
relevant_adversarial_probabilities
.shape[1]), self.testset.
labels[:relevant_adversarial_probabilities
.shape[1]], ] = 0
assert len(relevant_adversarial_probabilities.shape) == 3
adversarial_errors = -numpy.max(relevant_adversarial_probabilities,
axis=2)
total_adversarial_probabilities = common.numpy.concatenate(
total_adversarial_probabilities,
adversarial_probabilities,
axis=0)
total_adversarial_errors = common.numpy.concatenate(
total_adversarial_errors, adversarial_errors, axis=0)
eval = common.eval.AdversarialEvaluation(
clean_probabilities,
total_adversarial_probabilities,
self.testset.labels,
validation=0.9,
errors=total_adversarial_errors)
log('test error in %%: %g' % (eval.test_error() * 100))
log('test error @99%%tpr in %%: %g' %
(eval.test_error_at_99tpr() * 100))
log('robust test error in %%: %g' % (eval.robust_test_error() * 100))
log('robust test error @99%%tpr in %%: %g' %
(eval.robust_test_error_at_99tpr() * 100))
attack.initialization = attacks.initializations.LInfUniformNormInitialization(epsilon)
attack.projection = attacks.projections.SequentialProjections([
attacks.projections.LInfProjection(epsilon),
attacks.projections.BoxProjection()
])
attack.norm = attacks.norms.LInfNorm()
objective = attacks.objectives.UntargetedF0Objective()
model.eval()
# Obtain predicted probabilities on clean examples.
# Note that the full test set is used.
clean_probabilities = common.test.test(model, testloader, cuda=cuda)
# Run attack, will also provide the corresponding predicted probabilities.
# Note that only 1000 examples are attacked.
_, adversarial_probabilities, adversarial_errors = common.test.attack(model, adversarialloader, attack,
objective, attempts=5, cuda=cuda)
print(clean_probabilities.shape) # 10000 x 10
print(adversarial_probabilities.shape) # 5 x 100 x 10
# Use validation=0.1 such that 10% of the clean probabilities are
# used to determine the confidence threshold.
eval = common.eval.AdversarialEvaluation(clean_probabilities, adversarial_probabilities,
testset.labels, validation=0.1, errors=adversarial_errors)
print('test error in %%: %g' % eval.test_error())
print('robust test error in %%: %g' % eval.robust_test_error())
print('confidence threshold @99%%TPR: %g' % eval.confidence_at_99tpr())
print('test error @99%%TPR in %%: %g' % eval.test_error_at_99tpr())
print('false positive rate @99%%TPR in %%: %g' % eval.fpr_at_99tpr())
print('robust test error @99%%TPR in %%: %g' % eval.robust_test_error_at_99tpr())
def testNormalTraining(self):
optimizer = torch.optim.SGD(self.model.parameters(),
lr=0.1,
momentum=0.9)
scheduler = common.train.get_exponential_scheduler(
optimizer, batches_per_epoch=len(self.trainset))
writer = common.summary.SummaryDictWriter()
augmentation = None
trainer = common.train.NormalTraining(self.model,
self.trainset,
self.testset,
optimizer,
scheduler,
augmentation=augmentation,
writer=writer,
cuda=self.cuda)
trainer.summary_gradients = False
epochs = 10
for e in range(epochs):
trainer.step(e)
probabilities = common.test.test(self.model,
self.testset,
cuda=self.cuda)
eval = common.eval.CleanEvaluation(probabilities,
self.testset.dataset.labels,
validation=0)
self.assertGreaterEqual(0.05, eval.test_error())
epsilon = 0.3
attack = attacks.BatchGradientDescent()
attack.max_iterations = 2
attack.base_lr = 0.1
attack.momentum = 0
attack.c = 0
attack.lr_factor = 1.5
attack.normalized = True
attack.backtrack = True
attack.initialization = attacks.initializations.LInfUniformInitialization(
epsilon)
attack.norm = attacks.norms.LInfNorm()
attack.projection = attacks.projections.SequentialProjections([
attacks.projections.LInfProjection(epsilon),
attacks.projections.BoxProjection()
])
objective = attacks.objectives.UntargetedF7PObjective()
distal_attack = attacks.BatchGradientDescent()
distal_attack.max_iterations = 2
distal_attack.base_lr = 0.1
distal_attack.momentum = 0
distal_attack.c = 0
distal_attack.lr_factor = 1.5
distal_attack.normalized = True
distal_attack.backtrack = True
distal_attack.initialization = attacks.initializations.RandomInitializations(
[
attacks.initializations.LInfUniformNormInitialization(
epsilon), # !
attacks.initializations.SequentialInitializations([
attacks.initializations.LInfUniformNormInitialization(
epsilon), # !
attacks.initializations.SmoothInitialization()
])
])
distal_attack.norm = attacks.norms.LInfNorm()
distal_attack.projection = attacks.projections.SequentialProjections([
attacks.projections.LInfProjection(epsilon),
attacks.projections.BoxProjection()
])
distal_objective = attacks.objectives.UntargetedF0Objective(
loss=common.torch.max_log_loss)
adversarial_perturbations, adversarial_probabilities, _ = common.test.attack(
self.model,
self.adversarialset,
attack,
objective,
attempts=1,
writer=writer,
cuda=self.cuda)
eval = common.eval.AdversarialEvaluation(
probabilities[:len(self.adversarialset.dataset)],
adversarial_probabilities,
self.adversarialset.dataset.labels,
validation=0)
self.assertGreaterEqual(0.8,
eval.receiver_operating_characteristic_auc())
distal_perturbations, distal_probabilities, _ = common.test.attack(
self.model,
self.randomset,
distal_attack,
distal_objective,
attempts=1,
writer=writer,
cuda=self.cuda)
eval = common.eval.DistalEvaluation(
probabilities[:len(self.adversarialset.dataset)],
distal_probabilities,
self.adversarialset.dataset.labels,
validation=0)
self.assertGreaterEqual(0.8,
eval.receiver_operating_characteristic_auc())
def testConfidenceCalibratedAdversarialTraining(self):
optimizer = torch.optim.SGD(self.model.parameters(),
lr=0.1,
momentum=0.9)
scheduler = common.train.get_exponential_scheduler(
optimizer, batches_per_epoch=len(self.trainset))
writer = common.summary.SummaryDictWriter()
augmentation = None
epsilon = 0.3
attack = attacks.BatchGradientDescent()
attack.max_iterations = 2
attack.base_lr = 0.1
attack.momentum = 0
attack.c = 0
attack.lr_factor = 1.5
attack.normalized = True
attack.backtrack = True
attack.initialization = attacks.initializations.LInfUniformInitialization(
epsilon)
attack.norm = attacks.norms.LInfNorm()
attack.projection = attacks.projections.SequentialProjections([
attacks.projections.LInfProjection(epsilon),
attacks.projections.BoxProjection()
])
objective = attacks.objectives.UntargetedF7PObjective()
loss = common.torch.cross_entropy_divergence
transition = common.utils.partial(common.torch.linear_transition,
norm=attack.norm)
trainer = common.train.ConfidenceCalibratedAdversarialTraining(
self.model,
self.trainset,
self.testset,
optimizer,
scheduler,
attack,
objective,
loss,
transition,
fraction=0.5,
augmentation=augmentation,
writer=writer,
cuda=self.cuda)
trainer.summary_gradients = False
epochs = 10
for e in range(epochs):
trainer.step(e)
probabilities = common.test.test(self.model,
self.testset,
cuda=self.cuda)
eval = common.eval.CleanEvaluation(probabilities,
self.testset.dataset.labels,
validation=0)
self.assertGreaterEqual(0.05, eval.test_error())
adversarial_perturbations, adversarial_probabilities, _ = common.test.attack(
self.model,
self.adversarialset,
attack,
objective,
attempts=1,
writer=writer,
cuda=self.cuda)
eval = common.eval.AdversarialEvaluation(
probabilities[:len(self.adversarialset.dataset)],
adversarial_probabilities,
self.adversarialset.dataset.labels,
validation=0)
self.assertGreaterEqual(eval.receiver_operating_characteristic_auc(),
0.95)
def testResNet(self):
model = models.ResNet(10, [3, 32, 32], channels=12, blocks=[3, 3, 3])
eval = self.testTrainModel(model)
self.assertGreaterEqual(0.05, eval.test_error())
def testWideResNet(self):
model = models.ResNet(10, [1, 28, 28], depth=28, width=10, channels=8)
eval = self.testTrainModel(model)
self.assertGreaterEqual(0.03, eval.test_error())
