def evaluate(self):
"""
Evaluate.
"""
self.model.eval()
clean_probabilities = common.test.test(self.model,
self.testloader,
cuda=self.args.cuda)
attack, objective = self.get_attack()
adversarial_perturbations, adversarial_probabilities, adversarial_errors = common.test.attack(
self.model,
self.adversarialloader,
attack,
objective,
attempts=5,
cuda=self.args.cuda)
eval = common.eval.AdversarialEvaluation(clean_probabilities,
adversarial_probabilities,
self.testset.labels,
validation=0.9,
errors=adversarial_errors)
log('test error in %%: %g' % (eval.test_error() * 100))
log('test error @99%%tpr in %%: %g' %
(eval.test_error_at_99tpr() * 100))
log('robust test error in %%: %g' % (eval.robust_test_error() * 100))
log('robust test error @99%%tpr in %%: %g' %
(eval.robust_test_error_at_99tpr() * 100))
def testReferenceAdversarialTraining(self):
optimizer = torch.optim.SGD(self.model.parameters(),
lr=0.1,
momentum=0.9)
scheduler = common.train.get_exponential_scheduler(
optimizer, batches_per_epoch=len(self.trainset))
writer = common.summary.SummaryDictWriter()
augmentation = None
epsilon = 0.3
attack = attacks.BatchReferencePGD()
attack.max_iterations = 10
attack.epsilon = epsilon
attack.base_lr = 0.1
attack.norm = attacks.norms.LInfNorm()
objective = attacks.objectives.UntargetedF0Objective()
trainer = common.train.AdversarialTraining(self.model,
self.trainset,
self.testset,
optimizer,
scheduler,
attack,
objective,
fraction=1,
augmentation=augmentation,
writer=writer,
cuda=self.cuda)
trainer.summary_gradients = False
epochs = 10
for e in range(epochs):
trainer.step(e)
probabilities = common.test.test(self.model,
self.testset,
cuda=self.cuda)
eval = common.eval.CleanEvaluation(probabilities,
self.testset.dataset.labels,
validation=0)
self.assertGreaterEqual(0.05, eval.test_error())
adversarial_perturbations, adversarial_probabilities, _ = common.test.attack(
self.model,
self.adversarialset,
attack,
objective,
attempts=1,
writer=writer,
cuda=self.cuda)
eval = common.eval.AdversarialEvaluation(
probabilities[:len(self.adversarialset.dataset)],
adversarial_probabilities,
self.adversarialset.dataset.labels,
validation=0)
self.assertGreaterEqual(0.2, eval.robust_test_error())
def evaluate(self):
"""
Evaluate.
"""
# это не работает и это нормально. так и должно быть. для него надо отдельно код фиксить
self.model.eval()
clean_probabilities = common.test.test(self.model,
self.testloader,
cuda=True)
total_adversarial_probabilities = None
total_adversarial_errors = None
for attack, objective, attempts in self.get_test_attacks():
_, adversarial_probabilities, _ = common.test.attack(
self.model,
self.adversarialloader,
attack,
objective,
attempts=attempts,
cuda=True)
relevant_adversarial_probabilities = numpy.copy(
adversarial_probabilities)
relevant_adversarial_probabilities[:,
numpy.arange(
relevant_adversarial_probabilities
.shape[1]), self.testset.
labels[:relevant_adversarial_probabilities
.shape[1]], ] = 0
assert len(relevant_adversarial_probabilities.shape) == 3
adversarial_errors = -numpy.max(relevant_adversarial_probabilities,
axis=2)
total_adversarial_probabilities = common.numpy.concatenate(
total_adversarial_probabilities,
adversarial_probabilities,
axis=0)
total_adversarial_errors = common.numpy.concatenate(
total_adversarial_errors, adversarial_errors, axis=0)
eval = common.eval.AdversarialEvaluation(
clean_probabilities,
total_adversarial_probabilities,
self.testset.labels,
validation=0.9,
errors=total_adversarial_errors)
log('test error in %%: %g' % (eval.test_error() * 100))
log('test error @99%%tpr in %%: %g' %
(eval.test_error_at_99tpr() * 100))
log('robust test error in %%: %g' % (eval.robust_test_error() * 100))
log('robust test error @99%%tpr in %%: %g' %
(eval.robust_test_error_at_99tpr() * 100))
log(f'success rate: {eval.success_rate()}')
def evaluate(self):
"""
Evaluate.
"""
self.model.eval()
clean_probabilities = common.test.test(self.model,
self.testloader,
cuda=self.args.cuda)
total_adversarial_probabilities = None
total_adversarial_errors = None
for attack, objective, attempts in self.get_test_attacks():
_, adversarial_probabilities, _ = common.test.attack(
self.model,
self.adversarialloader,
attack,
objective,
attempts=attempts,
cuda=self.args.cuda)
relevant_adversarial_probabilities = numpy.copy(
adversarial_probabilities)
relevant_adversarial_probabilities[:,
numpy.arange(
relevant_adversarial_probabilities
.shape[1]), self.testset.
labels[:relevant_adversarial_probabilities
.shape[1]], ] = 0
assert len(relevant_adversarial_probabilities.shape) == 3
adversarial_errors = -numpy.max(relevant_adversarial_probabilities,
axis=2)
total_adversarial_probabilities = common.numpy.concatenate(
total_adversarial_probabilities,
adversarial_probabilities,
axis=0)
total_adversarial_errors = common.numpy.concatenate(
total_adversarial_errors, adversarial_errors, axis=0)
eval = common.eval.AdversarialEvaluation(
clean_probabilities,
total_adversarial_probabilities,
self.testset.labels,
validation=0.9,
errors=total_adversarial_errors)
log('test error in %%: %g' % (eval.test_error() * 100))
log('test error @99%%tpr in %%: %g' %
(eval.test_error_at_99tpr() * 100))
log('robust test error in %%: %g' % (eval.robust_test_error() * 100))
log('robust test error @99%%tpr in %%: %g' %
(eval.robust_test_error_at_99tpr() * 100))
attack.initialization = attacks.initializations.LInfUniformNormInitialization(epsilon)
attack.projection = attacks.projections.SequentialProjections([
attacks.projections.LInfProjection(epsilon),
attacks.projections.BoxProjection()
])
attack.norm = attacks.norms.LInfNorm()
objective = attacks.objectives.UntargetedF0Objective()
model.eval()
# Obtain predicted probabilities on clean examples.
# Note that the full test set is used.
clean_probabilities = common.test.test(model, testloader, cuda=cuda)
# Run attack, will also provide the corresponding predicted probabilities.
# Note that only 1000 examples are attacked.
_, adversarial_probabilities, adversarial_errors = common.test.attack(model, adversarialloader, attack,
objective, attempts=5, cuda=cuda)
print(clean_probabilities.shape) # 10000 x 10
print(adversarial_probabilities.shape) # 5 x 100 x 10
# Use validation=0.1 such that 10% of the clean probabilities are
# used to determine the confidence threshold.
eval = common.eval.AdversarialEvaluation(clean_probabilities, adversarial_probabilities,
testset.labels, validation=0.1, errors=adversarial_errors)
print('test error in %%: %g' % eval.test_error())
print('robust test error in %%: %g' % eval.robust_test_error())
print('confidence threshold @99%%TPR: %g' % eval.confidence_at_99tpr())
print('test error @99%%TPR in %%: %g' % eval.test_error_at_99tpr())
print('false positive rate @99%%TPR in %%: %g' % eval.fpr_at_99tpr())
print('robust test error @99%%TPR in %%: %g' % eval.robust_test_error_at_99tpr())
l0_attack.initialization = attacks.initializations.L1UniformNormInitialization(
l0_epsilon)
l0_attack.projection = attacks.projections.SequentialProjections([
attacks.projections.L0Projection(l0_epsilon),
attacks.projections.BoxProjection()
])
l0_attack.norm = attacks.norms.L0Norm()
objective = attacks.objectives.UntargetedF0Objective()
labels = ['linf', 'l2', 'l1', 'l0']
epsilons = [linf_epsilon, l2_epsilon, l1_epsilon, l0_epsilon]
attacks = [linf_attack, l2_attack, l1_attack, l0_attack]
model.eval()
clean_probabilities = common.test.test(model, testloader, cuda=cuda)
for a in range(len(attacks)):
_, adversarial_probabilities, _ = common.test.attack(model,
adversarialloader,
linf_attack,
objective,
attempts=1,
cuda=cuda)
eval = common.eval.AdversarialEvaluation(clean_probabilities,
adversarial_probabilities,
testset.labels,
validation=0.1)
print('[%s, epsilon=%g] robust test error in %%: %g' %
(labels[a], epsilons[a], (100 * eval.robust_test_error())))
print('[%s, epsilon=%g] robust test error @99%%TPR in %%: %g' %
(labels[a], epsilons[a], (100 * eval.robust_test_error_at_99tpr())))
