def test_keras_mnist(self):
"""
Second test with the KerasClassifier.
:return:
"""
(_, _), (x_test, _) = self.mnist
x_test_original = x_test.copy()
# Build KerasClassifier
krc = get_classifier_kr()
# Attack
nf = NewtonFool(krc, max_iter=5, batch_size=100)
x_test_adv = nf.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
y_pred = krc.predict(x_test)
y_pred_adv = krc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= .9 * y_pred_adv_max).all())
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))),
0.0,
delta=0.00001)
def test_pytorch_mnist(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
(_, _), (x_test, _) = self.mnist
x_test = np.swapaxes(x_test, 1, 3).astype(np.float32)
x_test_original = x_test.copy()
# Build PyTorchClassifier
ptc = get_classifier_pt()
# Attack
nf = NewtonFool(ptc, max_iter=5, batch_size=100)
x_test_adv = nf.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
y_pred = ptc.predict(x_test)
y_pred_adv = ptc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= .9 * y_pred_adv_max).all())
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))),
0.0,
delta=0.00001)
def test_scikitlearn(self):
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from art.classifiers.scikitlearn import ScikitlearnLogisticRegression, ScikitlearnSVC
scikitlearn_test_cases = {LogisticRegression: ScikitlearnLogisticRegression} # ,
# SVC: ScikitlearnSVC,
# LinearSVC: ScikitlearnSVC}
(_, _), (x_test, y_test) = self.iris
for (model_class, classifier_class) in scikitlearn_test_cases.items():
model = model_class()
classifier = classifier_class(model=model, clip_values=(0, 1))
classifier.fit(x=x_test, y=y_test)
attack = NewtonFool(classifier, max_iter=5, batch_size=128)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy of ' + classifier.__class__.__name__ + ' on Iris with NewtonFool adversarial examples'
': %.2f%%', (acc * 100))
def test_tensorflow_mnist(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
x_test_original = self.x_test_mnist.copy()
# Build TensorFlowClassifier
tfc, sess = get_image_classifier_tf()
# Attack
nf = NewtonFool(tfc, max_iter=5, batch_size=100)
x_test_adv = nf.generate(self.x_test_mnist)
self.assertFalse((self.x_test_mnist == x_test_adv).all())
y_pred = tfc.predict(self.x_test_mnist)
y_pred_adv = tfc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= 0.9 * y_pred_adv_max).all())
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(
np.max(np.abs(x_test_original - self.x_test_mnist))),
0.0,
delta=0.00001)
def atk_NeutonFool(x_train, x_test, y_train, y_test, classifier):
adv_crafter = NewtonFool(classifier, max_iter=20)
x_train_adv = adv_crafter.generate(x_train)
x_test_adv = adv_crafter.generate(x_test)
print("After NeutonFool Attack \n")
evaluate(x_train, x_test, y_train, y_test, x_train_adv, x_test_adv, classifier)
return x_test_adv, x_train_adv
def newton_fool(x_test, model, max_iter, eta, batch_size):
classifier = KerasClassifier(model=model, clip_values=(0, 1))
attack_cw = NewtonFool(classifier,
max_iter=max_iter,
eta=eta,
batch_size=batch_size)
x_test_adv = attack_cw.generate(x_test)
return np.reshape(x_test_adv, (32, 32, 3))
def test_iris_pt(self):
(_, _), (x_test, y_test) = self.iris
classifier = get_iris_classifier_pt()
attack = NewtonFool(classifier, max_iter=5, batch_size=128)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with NewtonFool adversarial examples: %.2f%%', (acc * 100))
def test_iris_k_unbounded(self):
(_, _), (x_test, y_test) = self.iris
classifier, _ = get_iris_classifier_kr()
# Recreate a classifier without clip values
classifier = KerasClassifier(model=classifier._model, use_logits=False, channel_index=1)
attack = NewtonFool(classifier, max_iter=5, batch_size=128)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy on Iris with NewtonFool adversarial examples: %.2f%%', (acc * 100))
def test_pytorch_iris(self):
classifier = get_tabular_classifier_pt()
attack = NewtonFool(classifier, max_iter=5, batch_size=128)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris,
axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(
self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Accuracy on Iris with NewtonFool adversarial examples: %.2f%%",
(acc * 100))
def GetAttackers(classifier, x_test, attacker_name):
"""
Function:
Load classifier and generate adversarial samples
"""
t_start = time.time()
if attacker_name == "FGSM":
attacker = FastGradientMethod(classifier=classifier, eps=0.3)
elif attacker_name == "Elastic":
attacker = ElasticNet(classifier=classifier, confidence=0.5)
elif attacker_name == "BasicIterativeMethod":
attacker = BasicIterativeMethod(classifier=classifier, eps=0.3)
elif attacker_name == "NewtonFool":
attacker = NewtonFool(classifier=classifier, max_iter=20)
elif attacker_name == "HopSkipJump":
attacker = HopSkipJump(classifier=classifier, max_iter=20)
elif attacker_name == "ZooAttack":
attacker = ZooAttack(classifier=classifier, max_iter=20)
elif attacker_name == "VirtualAdversarialMethod":
attacker = VirtualAdversarialMethod(classifier=classifier, max_iter=20)
elif attacker_name == "UniversalPerturbation":
attacker = UniversalPerturbation(classifier=classifier, max_iter=20)
elif attacker_name == "AdversarialPatch":
attacker = AdversarialPatch(classifier=classifier, max_iter=20)
elif attacker_name == "Attack":
attacker = Attack(classifier=classifier)
elif attacker_name == "BoundaryAttack":
attacker = BoundaryAttack(classifier=classifier,
targeted=False,
epsilon=0.05,
max_iter=20) #, max_iter=20
elif attacker_name == "CarliniL2":
attacker = CarliniL2Method(classifier=classifier,
confidence=0.5,
learning_rate=0.001,
max_iter=15)
elif attacker_name == "CarliniLinf":
attacker = CarliniLInfMethod(classifier=classifier,
confidence=0.5,
learning_rate=0.001,
max_iter=15)
elif attacker_name == "DeepFool":
attacker = DeepFool(classifier)
elif attacker_name == "SMM":
attacker = SaliencyMapMethod(classifier=classifier, theta=2)
elif attacker_name == "PGD":
attacker = ProjectedGradientDescent(classifier=classifier,
norm=2,
eps=1,
eps_step=0.5)
else:
raise ValueError("Please get the right attacker's name for the input.")
test_adv = attacker.generate(x_test)
dt = time.time() - t_start
return test_adv, dt
def test_scikitlearn(self):
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from art.classifiers.scikitlearn import SklearnClassifier
scikitlearn_test_cases = [
LogisticRegression(solver="lbfgs", multi_class="auto"),
SVC(gamma="auto"),
LinearSVC(),
]
x_test_original = self.x_test_iris.copy()
for model in scikitlearn_test_cases:
classifier = SklearnClassifier(model=model, clip_values=(0, 1))
classifier.fit(x=self.x_test_iris, y=self.y_test_iris)
attack = NewtonFool(classifier, max_iter=5, batch_size=128)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris,
axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(
self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Accuracy of " + classifier.__class__.__name__ +
" on Iris with NewtonFool adversarial examples"
": %.2f%%",
(acc * 100),
)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(
np.max(np.abs(x_test_original - self.x_test_iris))),
0.0,
delta=0.00001)
def test_scikitlearn(self):
from sklearn.linear_model import LogisticRegression
from art.classifiers.scikitlearn import ScikitlearnLogisticRegression
scikitlearn_test_cases = {
LogisticRegression: ScikitlearnLogisticRegression
} # ,
# SVC: ScikitlearnSVC,
# LinearSVC: ScikitlearnSVC}
(_, _), (x_test, y_test) = self.iris
x_test_original = x_test.copy()
for (model_class, classifier_class) in scikitlearn_test_cases.items():
model = model_class()
classifier = classifier_class(model=model, clip_values=(0, 1))
classifier.fit(x=x_test, y=y_test)
attack = NewtonFool(classifier, max_iter=5, batch_size=128)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(
preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info(
'Accuracy of ' + classifier.__class__.__name__ +
' on Iris with NewtonFool adversarial examples'
': %.2f%%', (acc * 100))
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(
np.max(np.abs(x_test_original - x_test))),
0.0,
delta=0.00001)
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Build KerasClassifier
krc = get_classifier_kr()
# Get MNIST
(_, _), (x_test, _) = self.mnist
# Attack
nf = NewtonFool(krc, max_iter=5, batch_size=100)
x_test_adv = nf.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
y_pred = krc.predict(x_test)
y_pred_adv = krc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= .9 * y_pred_adv_max).all())
def test_tfclassifier(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
# Build TensorFlowClassifier
tfc, sess = get_classifier_tf()
# Get MNIST
(_, _), (x_test, _) = self.mnist
# Attack
nf = NewtonFool(tfc, max_iter=5, batch_size=100)
x_test_adv = nf.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
y_pred = tfc.predict(x_test)
y_pred_adv = tfc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= .9 * y_pred_adv_max).all())
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
# Get MNIST
(_, _), (x_test, _) = self.mnist
x_test = np.swapaxes(x_test, 1, 3).astype(np.float32)
# Attack
nf = NewtonFool(ptc, max_iter=5, batch_size=100)
x_test_adv = nf.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
y_pred = ptc.predict(x_test)
y_pred_adv = ptc.predict(x_test_adv)
y_pred_bool = y_pred.max(axis=1, keepdims=1) == y_pred
y_pred_max = y_pred.max(axis=1)
y_pred_adv_max = y_pred_adv[y_pred_bool]
self.assertTrue((y_pred_max >= .9 * y_pred_adv_max).all())
def make_attack(x, y, model, name, style='PGD'):
if style == 'PGD':
attacked_data = np.array([
PGD_infini(x[idx], y[idx], cst.PGD_attack_delta,
cst.PGD_attack_epsilon, cst.PGD_attack_nb_iter, model)
for idx in tqdm_notebook(range(len(x)))
])
np.save(os.path.join(cst.DATA, name), attacked_data)
if style == 'FGSM':
attacked_data = np.array([
FGSM(x[idx], y[idx], cst.FGSM_attack_delta, model)
for idx in tqdm_notebook(range(len(x)))
])
np.save(os.path.join(cst.DATA, name), attacked_data)
if style == 'Carlini_Inf':
attacked_data = np.array([
carlini_inf(x[idx], model, cst.CInf_eps, cst.CInf_max_iter,
cst.CInf_learning_rate)
for idx in tqdm_notebook(range(len(x)))
])
np.save(os.path.join(cst.DATA, name), attacked_data)
if style == 'Deep_Fool':
attacked_data = np.array([
deep_fool(x[idx], model, cst.DFool_max_iter, cst.DFool_epsilon,
cst.DFool_nb_grads, cst.DFool_batch_size)
for idx in tqdm_notebook(range(len(x)))
])
np.save(os.path.join(cst.DATA, name), attacked_data)
if style == 'Newton_Fool':
attacked_data = np.array([
NewtonFool(x[idx], model, cst.NFool_max_iter, cst.NFool_eta,
cst.NFool_batch_size)
for idx in tqdm_notebook(range(len(x)))
])
np.save(os.path.join(cst.DATA, name), attacked_data)
if style == 'Basic_Iterative_Method':
attacked_data = np.array([
basic_iter(x[idx], model, cst.BIter_eps, cst.BIter_eps_step,
cst.BIter_max_iter, cst.BIter_targeted,
cst.BIter_batch_size)
for idx in tqdm_notebook(range(len(x)))
])
np.save(os.path.join(cst.DATA, name), attacked_data)
return attacked_data