def GetAttackers(classifier, x_test, attacker_name):
"""
Function:
Load classifier and generate adversarial samples
"""
t_start = time.time()
if attacker_name == "FGSM":
attacker = FastGradientMethod(classifier=classifier, eps=0.3)
elif attacker_name == "Elastic":
attacker = ElasticNet(classifier=classifier, confidence=0.5)
elif attacker_name == "BasicIterativeMethod":
attacker = BasicIterativeMethod(classifier=classifier, eps=0.3)
elif attacker_name == "NewtonFool":
attacker = NewtonFool(classifier=classifier, max_iter=20)
elif attacker_name == "HopSkipJump":
attacker = HopSkipJump(classifier=classifier, max_iter=20)
elif attacker_name == "ZooAttack":
attacker = ZooAttack(classifier=classifier, max_iter=20)
elif attacker_name == "VirtualAdversarialMethod":
attacker = VirtualAdversarialMethod(classifier=classifier, max_iter=20)
elif attacker_name == "UniversalPerturbation":
attacker = UniversalPerturbation(classifier=classifier, max_iter=20)
elif attacker_name == "AdversarialPatch":
attacker = AdversarialPatch(classifier=classifier, max_iter=20)
elif attacker_name == "Attack":
attacker = Attack(classifier=classifier)
elif attacker_name == "BoundaryAttack":
attacker = BoundaryAttack(classifier=classifier,
targeted=False,
epsilon=0.05,
max_iter=20) #, max_iter=20
elif attacker_name == "CarliniL2":
attacker = CarliniL2Method(classifier=classifier,
confidence=0.5,
learning_rate=0.001,
max_iter=15)
elif attacker_name == "CarliniLinf":
attacker = CarliniLInfMethod(classifier=classifier,
confidence=0.5,
learning_rate=0.001,
max_iter=15)
elif attacker_name == "DeepFool":
attacker = DeepFool(classifier)
elif attacker_name == "SMM":
attacker = SaliencyMapMethod(classifier=classifier, theta=2)
elif attacker_name == "PGD":
attacker = ProjectedGradientDescent(classifier=classifier,
norm=2,
eps=1,
eps_step=0.5)
else:
raise ValueError("Please get the right attacker's name for the input.")
test_adv = attacker.generate(x_test)
dt = time.time() - t_start
return test_adv, dt
def test_pytorch_resume(self):
x_test = np.reshape(self.x_test_mnist, (self.x_test_mnist.shape[0], 1, 28, 28)).astype(np.float32)
# Build PyTorchClassifier
ptc = get_image_classifier_pt()
# HSJ attack
hsj = HopSkipJump(classifier=ptc, targeted=True, max_iter=10, max_eval=100, init_eval=10)
params = {"y": self.y_test_mnist[2:3], "x_adv_init": x_test[2:3]}
x_test_adv1 = hsj.generate(x_test[0:1], **params)
diff1 = np.linalg.norm(x_test_adv1 - x_test)
params.update(resume=True, x_adv_init=x_test_adv1)
x_test_adv2 = hsj.generate(x_test[0:1], **params)
params.update(x_adv_init=x_test_adv2)
x_test_adv2 = hsj.generate(x_test[0:1], **params)
diff2 = np.linalg.norm(x_test_adv2 - x_test)
self.assertGreater(diff1, diff2)
def test_classifier_type_check_fail_classifier(self):
# Use a useless test classifier to test basic classifier properties
class ClassifierNoAPI:
pass
classifier = ClassifierNoAPI
with self.assertRaises(TypeError) as context:
_ = HopSkipJump(classifier=classifier)
self.assertIn('For `HopSkipJump` classifier must be an instance of `art.classifiers.classifier.Classifier`, the'
' provided classifier is instance of (<class \'object\'>,).', str(context.exception))
def test_pytorch_iris(self):
classifier = get_tabular_classifier_pt()
x_test = self.x_test_iris.astype(np.float32)
# Norm=2
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
# Norm=np.inf
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
def test_keras_iris_clipped(self):
classifier = get_tabular_classifier_kr()
# Norm=2
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
# Norm=np.inf
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
# Clean-up session
k.clear_session()
def test_scikitlearn(self):
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier
from sklearn.ensemble import AdaBoostClassifier, BaggingClassifier, ExtraTreesClassifier
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from art.classifiers.scikitlearn import ScikitlearnDecisionTreeClassifier, ScikitlearnExtraTreeClassifier
from art.classifiers.scikitlearn import ScikitlearnAdaBoostClassifier, ScikitlearnBaggingClassifier
from art.classifiers.scikitlearn import ScikitlearnExtraTreesClassifier, ScikitlearnGradientBoostingClassifier
from art.classifiers.scikitlearn import ScikitlearnRandomForestClassifier, ScikitlearnLogisticRegression
from art.classifiers.scikitlearn import ScikitlearnSVC
scikitlearn_test_cases = {DecisionTreeClassifier: ScikitlearnDecisionTreeClassifier,
ExtraTreeClassifier: ScikitlearnExtraTreeClassifier,
AdaBoostClassifier: ScikitlearnAdaBoostClassifier,
BaggingClassifier: ScikitlearnBaggingClassifier,
ExtraTreesClassifier: ScikitlearnExtraTreesClassifier,
GradientBoostingClassifier: ScikitlearnGradientBoostingClassifier,
RandomForestClassifier: ScikitlearnRandomForestClassifier,
LogisticRegression: ScikitlearnLogisticRegression,
SVC: ScikitlearnSVC,
LinearSVC: ScikitlearnSVC}
(_, _), (x_test, y_test) = self.iris
x_test_original = x_test.copy()
for (model_class, classifier_class) in scikitlearn_test_cases.items():
model = model_class()
classifier = classifier_class(model=model, clip_values=(0, 1))
classifier.fit(x=x_test, y=y_test)
# Norm=2
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy of ' + classifier.__class__.__name__ + ' on Iris with HopSkipJump adversarial '
'examples: %.2f%%', (acc * 100))
# Norm=np.inf
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info('Accuracy of ' + classifier.__class__.__name__ + ' on Iris with HopSkipJump adversarial '
'examples: %.2f%%', (acc * 100))
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def test_keras_iris_unbounded(self):
classifier = get_tabular_classifier_kr()
# Recreate a classifier without clip values
classifier = KerasClassifier(model=classifier._model, use_logits=False, channel_index=1)
# Norm=2
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
# Norm=np.inf
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
# Clean-up session
k.clear_session()
def test_iris_pt(self):
(_, _), (x_test, y_test) = self.iris
classifier = get_iris_classifier_pt()
# Norm=2
attack = HopSkipJump(classifier,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info(
'Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%',
(acc * 100))
# Norm=np.inf
attack = HopSkipJump(classifier,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info(
'Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%',
(acc * 100))
def test_ptclassifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
# Build PyTorchClassifier
ptc = get_classifier_pt()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
x_test = np.swapaxes(x_test, 1, 3).astype(np.float32)
# First targeted attack and norm=2
hsj = HopSkipJump(classifier=ptc,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10)
params = {'y': random_targets(y_test, ptc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# First targeted attack and norm=np.inf
hsj = HopSkipJump(classifier=ptc,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
params = {'y': random_targets(y_test, ptc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second untargeted attack and norm=2
hsj = HopSkipJump(classifier=ptc,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(ptc.predict(x_test), axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Second untargeted attack and norm=np.inf
hsj = HopSkipJump(classifier=ptc,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(ptc.predict(x_test), axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
def test_krclassifier(self):
"""
Second test with the KerasClassifier.
:return:
"""
# Build KerasClassifier
krc = get_classifier_kr()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# First targeted attack and norm=2
hsj = HopSkipJump(classifier=krc,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10)
params = {'y': random_targets(y_test, krc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# First targeted attack and norm=np.inf
hsj = HopSkipJump(classifier=krc,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
params = {'y': random_targets(y_test, krc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second untargeted attack and norm=2
hsj = HopSkipJump(classifier=krc,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(krc.predict(x_test), axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Second untargeted attack and norm=np.inf
hsj = HopSkipJump(classifier=krc,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(krc.predict(x_test), axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Clean-up session
k.clear_session()
tf.reset_default_graph()
def test_tensorflow_mnist(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
x_test_original = self.x_test_mnist.copy()
# Build TensorFlowClassifier
tfc, sess = get_image_classifier_tf()
# First targeted attack and norm=2
hsj = HopSkipJump(classifier=tfc, targeted=True, max_iter=2, max_eval=100, init_eval=10)
params = {"y": random_targets(self.y_test_mnist, tfc.nb_classes())}
x_test_adv = hsj.generate(self.x_test_mnist, **params)
self.assertFalse((self.x_test_mnist == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params["y"], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# First targeted attack and norm=np.inf
hsj = HopSkipJump(classifier=tfc, targeted=True, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
params = {"y": random_targets(self.y_test_mnist, tfc.nb_classes())}
x_test_adv = hsj.generate(self.x_test_mnist, **params)
self.assertFalse((self.x_test_mnist == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params["y"], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second untargeted attack and norm=2
hsj = HopSkipJump(classifier=tfc, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = hsj.generate(self.x_test_mnist)
self.assertFalse((self.x_test_mnist == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(tfc.predict(self.x_test_mnist), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Second untargeted attack and norm=np.inf
hsj = HopSkipJump(classifier=tfc, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = hsj.generate(self.x_test_mnist)
self.assertFalse((self.x_test_mnist == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(tfc.predict(self.x_test_mnist), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - self.x_test_mnist))), 0.0, delta=0.00001)
# Clean-up session
if sess is not None:
sess.close()
def test_scikitlearn(self):
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier
from sklearn.ensemble import AdaBoostClassifier, BaggingClassifier, ExtraTreesClassifier
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from art.classifiers.scikitlearn import SklearnClassifier
scikitlearn_test_cases = [
DecisionTreeClassifier(),
ExtraTreeClassifier(),
AdaBoostClassifier(),
BaggingClassifier(),
ExtraTreesClassifier(n_estimators=10),
GradientBoostingClassifier(n_estimators=10),
RandomForestClassifier(n_estimators=10),
LogisticRegression(solver="lbfgs", multi_class="auto"),
SVC(gamma="auto"),
LinearSVC(),
]
x_test_original = self.x_test_iris.copy()
for model in scikitlearn_test_cases:
classifier = SklearnClassifier(model=model, clip_values=(0, 1))
classifier.fit(x=self.x_test_iris, y=self.y_test_iris)
# Norm=2
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Accuracy of " + classifier.__class__.__name__ + " on Iris with HopSkipJump adversarial "
"examples: %.2f%%",
(acc * 100),
)
# Norm=np.inf
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Accuracy of " + classifier.__class__.__name__ + " on Iris with HopSkipJump adversarial "
"examples: %.2f%%",
(acc * 100),
)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - self.x_test_iris))), 0.0, delta=0.00001)
def adversarial_attack_shift(x, y, delta=1.0, model=RandomForestClassifier(), attack_type='zoo',
numerical_features=None, feat_delta=1.0):
# in this case delta is the portion of half the data on which to generate attacks
# because the first half as a minimum has to be used to train a model against which generate the attacks
assert (attack_type in ['zoo', 'boundary', 'hop-skip-jump'])
le = preprocessing.LabelEncoder()
le.fit(np.squeeze(y))
y = le.transform(y)
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=(0.5 * delta))
if numerical_features is not None:
n_numerical = len(numerical_features)
feat_indices = np.random.choice(n_numerical, ceil(n_numerical * feat_delta), replace=False)
feat_indices = np.array(numerical_features)[feat_indices]
else:
feat_indices = np.random.choice(x.shape[1], ceil(x.shape[1] * feat_delta), replace=False)
other_features = list(set(range(x.shape[1])) - set(feat_indices))
x_train_other = x_train[:, other_features]
x_train_numerical = x_train[:, feat_indices]
x_test_other = x_test[:, other_features]
x_test_numerical = x_test[:, feat_indices]
classifier = SklearnClassifier(model=model, clip_values=(0, np.max(x_train_numerical)))
# Train the ART classifier
classifier.fit(x_train_numerical, y_train)
# Evaluate the ART classifier on benign test examples
predictions = classifier.predict(x_test_numerical)
accuracy = np.sum(np.argmax(predictions, axis=1) == y_test) / len(y_test)
print("Accuracy on benign test examples: {}%".format(accuracy * 100))
# Generate adversarial test examples
if attack_type == 'zoo':
attack = ZooAttack(
classifier=classifier,
confidence=0.0,
targeted=False,
learning_rate=1e-1,
max_iter=10,
binary_search_steps=10,
initial_const=1e-3,
abort_early=True,
use_resize=False,
use_importance=False,
nb_parallel=x_test_numerical.shape[1],
batch_size=1,
variable_h=0.01,
)
elif attack_type == 'boundary':
attack = BoundaryAttack(classifier, targeted=False, epsilon=0.02, max_iter=20, num_trial=10)
elif attack_type == 'hop-skip-jump':
attack = HopSkipJump(classifier,
targeted=False,
norm=2,
max_iter=20,
max_eval=10,
init_eval=9,
init_size=10)
x_adv = attack.generate(x=x_test_numerical, y=y_test)
# Evaluate the ART classifier on adversarial test examples
predictions_adv = classifier.predict(x_adv)
accuracy = np.sum(np.argmax(predictions_adv, axis=1) == y_test) / len(y_test)
print("Accuracy on adversarial test examples: {}%".format(accuracy * 100))
print("Max difference: {}".format(np.max(np.abs(x_test_numerical - x_adv) / x_test_numerical)))
x_final = np.zeros_like(x)
x_final[:, feat_indices] = np.vstack([x_train_numerical, x_adv])
x_final[:, other_features] = np.vstack([x_train_other, x_test_other])
y_final = np.concatenate([y_train, y_test], axis=0)
y_final = le.inverse_transform(y_final)
adv_indices = list(range(len(y_train), len(y)))
return x_final, y_final, adv_indices, feat_indices
def test_pytorch_classifier(self):
"""
Third test with the PyTorchClassifier.
:return:
"""
x_test = np.swapaxes(self.x_test_mnist, 1, 3).astype(np.float32)
x_test_original = x_test.copy()
# Build PyTorchClassifier
ptc = get_image_classifier_pt()
# First targeted attack and norm=2
hsj = HopSkipJump(classifier=ptc, targeted=True, max_iter=2, max_eval=100, init_eval=10)
params = {"y": random_targets(self.y_test_mnist, ptc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params["y"], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# First targeted attack and norm=np.inf
hsj = HopSkipJump(classifier=ptc, targeted=True, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
params = {"y": random_targets(self.y_test_mnist, ptc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params["y"], axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second untargeted attack and norm=2
hsj = HopSkipJump(classifier=ptc, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(ptc.predict(x_test), axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Second untargeted attack and norm=np.inf
hsj = HopSkipJump(classifier=ptc, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(ptc.predict(x_test), axis=1)
y_pred_adv = np.argmax(ptc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def test_keras_mnist(self):
"""
Second test with the KerasClassifier.
:return:
"""
(_, _), (x_test, y_test) = self.mnist
x_test_original = x_test.copy()
# Build KerasClassifier
krc = get_classifier_kr()
# First targeted attack and norm=2
hsj = HopSkipJump(classifier=krc, targeted=True, max_iter=2, max_eval=100, init_eval=10)
params = {'y': random_targets(y_test, krc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# First targeted attack and norm=np.inf
hsj = HopSkipJump(classifier=krc, targeted=True, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
params = {'y': random_targets(y_test, krc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second untargeted attack and norm=2
hsj = HopSkipJump(classifier=krc, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(krc.predict(x_test), axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Second untargeted attack and norm=np.inf
hsj = HopSkipJump(classifier=krc, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(krc.predict(x_test), axis=1)
y_pred_adv = np.argmax(krc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
# Clean-up session
k.clear_session()
def test_iris_tf(self):
(_, _), (x_test, y_test) = self.iris
classifier, sess = get_iris_classifier_tf()
# Test untargeted attack and norm=2
attack = HopSkipJump(classifier,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info(
'Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%',
(acc * 100))
# Test untargeted attack and norm=np.inf
attack = HopSkipJump(classifier,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
x_test_adv = attack.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(y_test, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(y_test, axis=1)) / y_test.shape[0]
logger.info(
'Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%',
(acc * 100))
# Test targeted attack and norm=2
targets = random_targets(y_test, nb_classes=3)
attack = HopSkipJump(classifier,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10)
x_test_adv = attack.generate(x_test, **{'y': targets})
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertTrue((np.argmax(targets, axis=1) == preds_adv).any())
acc = np.sum(preds_adv == np.argmax(targets, axis=1)) / y_test.shape[0]
logger.info('Success rate of targeted HopSkipJump on Iris: %.2f%%',
(acc * 100))
# Test targeted attack and norm=np.inf
targets = random_targets(y_test, nb_classes=3)
attack = HopSkipJump(classifier,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
x_test_adv = attack.generate(x_test, **{'y': targets})
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertTrue((np.argmax(targets, axis=1) == preds_adv).any())
acc = np.sum(preds_adv == np.argmax(targets, axis=1)) / y_test.shape[0]
logger.info('Success rate of targeted HopSkipJump on Iris: %.2f%%',
(acc * 100))
# Clean-up session
sess.close()
def test_tfclassifier(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
# Build TensorFlowClassifier
tfc, sess = get_classifier_tf()
# Get MNIST
(_, _), (x_test, y_test) = self.mnist
# First targeted attack and norm=2
hsj = HopSkipJump(classifier=tfc,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10)
params = {'y': random_targets(y_test, tfc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# First targeted attack and norm=np.inf
hsj = HopSkipJump(classifier=tfc,
targeted=True,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
params = {'y': random_targets(y_test, tfc.nb_classes())}
x_test_adv = hsj.generate(x_test, **params)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
target = np.argmax(params['y'], axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((target == y_pred_adv).any())
# Second untargeted attack and norm=2
hsj = HopSkipJump(classifier=tfc,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(tfc.predict(x_test), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Second untargeted attack and norm=np.inf
hsj = HopSkipJump(classifier=tfc,
targeted=False,
max_iter=2,
max_eval=100,
init_eval=10,
norm=np.Inf)
x_test_adv = hsj.generate(x_test)
self.assertFalse((x_test == x_test_adv).all())
self.assertTrue((x_test_adv <= 1.0001).all())
self.assertTrue((x_test_adv >= -0.0001).all())
y_pred = np.argmax(tfc.predict(x_test), axis=1)
y_pred_adv = np.argmax(tfc.predict(x_test_adv), axis=1)
self.assertTrue((y_pred != y_pred_adv).any())
# Clean-up session
sess.close()
def test_tensorflow_iris(self):
classifier, sess = get_tabular_classifier_tf()
# Test untargeted attack and norm=2
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
# Test untargeted attack and norm=np.inf
attack = HopSkipJump(classifier, targeted=False, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info("Accuracy on Iris with HopSkipJump adversarial examples: %.2f%%", (acc * 100))
# Test targeted attack and norm=2
targets = random_targets(self.y_test_iris, nb_classes=3)
attack = HopSkipJump(classifier, targeted=True, max_iter=2, max_eval=100, init_eval=10)
x_test_adv = attack.generate(self.x_test_iris, **{"y": targets})
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertTrue((np.argmax(targets, axis=1) == preds_adv).any())
acc = np.sum(preds_adv == np.argmax(targets, axis=1)) / self.y_test_iris.shape[0]
logger.info("Success rate of targeted HopSkipJump on Iris: %.2f%%", (acc * 100))
# Test targeted attack and norm=np.inf
targets = random_targets(self.y_test_iris, nb_classes=3)
attack = HopSkipJump(classifier, targeted=True, max_iter=2, max_eval=100, init_eval=10, norm=np.Inf)
x_test_adv = attack.generate(self.x_test_iris, **{"y": targets})
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertTrue((np.argmax(targets, axis=1) == preds_adv).any())
acc = np.sum(preds_adv == np.argmax(targets, axis=1)) / self.y_test_iris.shape[0]
logger.info("Success rate of targeted HopSkipJump on Iris: %.2f%%", (acc * 100))
# Clean-up session
if sess is not None:
sess.close()
classifier = XGBoostClassifier(model=model, clip_values=(0, 1), nb_features=nfeatures, nb_classes=nclasses)
test_data, test_label = load_svmlight_file(data_path, n_features = nfeatures)
test_data = test_data.toarray()
test_label = test_label.astype('int')
n = len(test_label)
df = pd.DataFrame(test_data)
df['label'] = test_label
df = df.sample(frac=1)
test_label = df['label'].tolist()
test_data = np.array(df.drop(columns=['label']))
predictions = np.argmax(classifier.predict(test_data), axis=1)
attack = HopSkipJump(classifier=classifier, norm = np.inf)
n_selected = 100
corrected = []
c_labels = []
for i in range(len(test_label)):
if test_label[i] == predictions[i]:
corrected.append(test_data[i])
c_labels.append(test_label[i])
if len(corrected) >= n_selected:
break
corrected = np.array(corrected)
start = time.time()
test_adv = attack.generate(corrected)
end = time.time()
p_adv = np.argmax(classifier.predict(test_adv), axis=1)
print('Tesseract output is: ' + label_dict[np.argmax(classifier.predict(np.array([image_target])))])
"""
Tesseract output is: dissent
"""
# 1.5 this is the label we want to perturb to
plt.imshow(image_init)
plt.show()
print('Tesseract output is: ' + label_dict[np.argmax(classifier.predict(np.array([image_init])))])
"""
Tesseract output is: assent
"""
# 2. 使用HopSkipJump进行攻击
attack = HopSkipJump(classifier=classifier, targeted=True, norm=2, max_iter=0, max_eval=1000, init_eval=10)
iter_step = 10
x_adv = np.array([image_init])
for i in range(16):
x_adv = attack.generate(x=np.array([image_target]), x_adv_init=x_adv, y=to_categorical([1], 3))
#clear_output()
if i%3 == 0:
print("Adversarial image at step %d." % (i * iter_step), "L2 error",
np.linalg.norm(np.reshape(x_adv[0] - image_target, [-1])),
"and Tesseract output %s." % label_dict[np.argmax(classifier.predict(x_adv)[0])])
plt.imshow(x_adv[0])
plt.show(block=False)
attack.max_iter = iter_step
"""