def test_linearSVC(self):
"""
Test using a attack on LinearSVC
"""
(x_train, y_train), (x_test, y_test), min_, max_ = self.iris
x_test_original = x_test.copy()
# Build Scikitlearn Classifier
clip_values = (min_, max_)
clean = SklearnClassifier(model=LinearSVC(), clip_values=clip_values)
clean.fit(x_train, y_train)
poison = SklearnClassifier(model=LinearSVC(), clip_values=clip_values)
poison.fit(x_train, y_train)
attack = PoisoningAttackSVM(poison, 0.01, 1.0, x_train, y_train,
x_test, y_test, 100)
attack_y = np.array([1, 1]) - y_train[0]
attack_point, _ = attack.poison(np.array([x_train[0]]),
y=np.array([attack_y]))
poison.fit(x=np.vstack([x_train, attack_point]),
y=np.vstack([y_train,
np.copy(y_train[0].reshape((1, 2)))]))
acc = np.average(np.all(clean.predict(x_test) == y_test, axis=1)) * 100
poison_acc = np.average(
np.all(poison.predict(x_test) == y_test, axis=1)) * 100
logger.info("Clean Accuracy {}%".format(acc))
logger.info("Poison Accuracy {}%".format(poison_acc))
self.assertGreaterEqual(acc, poison_acc)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))),
0.0,
delta=0.00001)
def test_SVC_kernels(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
# Get MNIST
(x_train, y_train), (x_test, y_test), min_, max_ = self.iris
x_test_original = x_test.copy()
# Build Scikitlearn Classifier
clip_values = (min_, max_)
for kernel in ["linear"]: # ["linear", "poly", "rbf"]
clean = SklearnClassifier(model=SVC(kernel=kernel, gamma="auto"), clip_values=clip_values)
clean.fit(x_train, y_train)
poison = SklearnClassifier(model=SVC(kernel=kernel, gamma="auto"), clip_values=clip_values)
poison.fit(x_train, y_train)
attack = PoisoningAttackSVM(poison, 0.01, 1.0, x_train, y_train, x_test, y_test, 100)
attack_y = np.array([1, 1]) - y_train[0]
attack_point, _ = attack.poison(np.array([x_train[0]]), y=np.array([attack_y]))
poison.fit(
x=np.vstack([x_train, attack_point]),
y=np.vstack([y_train, np.array([1, 1]) - np.copy(y_train[0].reshape((1, 2)))]),
)
acc = np.average(np.all(clean.predict(x_test) == y_test, axis=1)) * 100
poison_acc = np.average(np.all(poison.predict(x_test) == y_test, axis=1)) * 100
logger.info("Clean Accuracy {}%".format(acc))
logger.info("Poison Accuracy {}%".format(poison_acc))
self.assertGreaterEqual(acc, poison_acc)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))), 0.0, delta=0.00001)
def test_scikitlearn(self):
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from art.estimators.classification.scikitlearn import SklearnClassifier
scikitlearn_test_cases = [
LogisticRegression(solver="lbfgs", multi_class="auto"),
SVC(gamma="auto"),
LinearSVC(),
]
x_test_original = self.x_test_iris.copy()
for model in scikitlearn_test_cases:
classifier = SklearnClassifier(model=model, clip_values=(0, 1))
classifier.fit(x=self.x_test_iris, y=self.y_test_iris)
# Test untargeted attack
attack = ElasticNet(classifier, targeted=False, max_iter=2)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
predictions_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris,
axis=1) == predictions_adv).all())
accuracy = 1.0 - np.sum(predictions_adv == np.argmax(
self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"EAD success rate of " + classifier.__class__.__name__ +
" on Iris: %.2f%%", (accuracy * 100))
# Test targeted attack
targets = random_targets(self.y_test_iris, nb_classes=3)
attack = ElasticNet(classifier, targeted=True, max_iter=2)
x_test_adv = attack.generate(self.x_test_iris, **{"y": targets})
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
predictions_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertTrue((np.argmax(targets,
axis=1) == predictions_adv).any())
accuracy = np.sum(predictions_adv == np.argmax(
targets, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Targeted EAD success rate of " +
classifier.__class__.__name__ + " on Iris: %.2f%%",
(accuracy * 100))
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(
np.max(np.abs(x_test_original - self.x_test_iris))),
0.0,
delta=0.00001)
def test_7_scikitlearn(self):
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from art.estimators.classification.scikitlearn import SklearnClassifier
scikitlearn_test_cases = [
LogisticRegression(solver="lbfgs", multi_class="auto"),
SVC(gamma="auto"),
LinearSVC(),
]
x_test_original = self.x_test_iris.copy()
for model in scikitlearn_test_cases:
classifier = SklearnClassifier(model=model, clip_values=(0, 1))
classifier.fit(x=self.x_test_iris, y=self.y_test_iris)
# Test untargeted attack
attack = ProjectedGradientDescent(classifier, eps=1.0, eps_step=0.1, max_iter=5, verbose=False)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Accuracy of " + classifier.__class__.__name__ + " on Iris with PGD adversarial examples: " "%.2f%%",
(acc * 100),
)
# Test targeted attack
targets = random_targets(self.y_test_iris, nb_classes=3)
attack = ProjectedGradientDescent(
classifier, targeted=True, eps=1.0, eps_step=0.1, max_iter=5, verbose=False
)
x_test_adv = attack.generate(self.x_test_iris, **{"y": targets})
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertTrue((np.argmax(targets, axis=1) == preds_adv).any())
acc = np.sum(preds_adv == np.argmax(targets, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Success rate of " + classifier.__class__.__name__ + " on targeted PGD on Iris: %.2f%%", (acc * 100)
)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - self.x_test_iris))), 0.0, delta=0.00001)
def test_scikitlearn(self):
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from art.estimators.classification.scikitlearn import SklearnClassifier
scikitlearn_test_cases = [
LogisticRegression(solver="lbfgs", multi_class="auto"),
SVC(gamma="auto"),
LinearSVC(),
]
x_test_original = self.x_test_iris.copy()
for model in scikitlearn_test_cases:
classifier = SklearnClassifier(model=model, clip_values=(0, 1))
classifier.fit(x=self.x_test_iris, y=self.y_test_iris)
attack = NewtonFool(classifier, max_iter=5, batch_size=128)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Accuracy of " + classifier.__class__.__name__ + " on Iris with NewtonFool adversarial examples"
": %.2f%%",
(acc * 100),
)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - self.x_test_iris))), 0.0, delta=0.00001)
def test_6_scikitlearn(self):
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier
from sklearn.ensemble import AdaBoostClassifier, BaggingClassifier, ExtraTreesClassifier
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from art.estimators.classification.scikitlearn import SklearnClassifier
scikitlearn_test_cases = [
DecisionTreeClassifier(),
ExtraTreeClassifier(),
AdaBoostClassifier(),
BaggingClassifier(),
ExtraTreesClassifier(n_estimators=10),
GradientBoostingClassifier(n_estimators=10),
RandomForestClassifier(n_estimators=10),
LogisticRegression(solver="lbfgs", multi_class="auto"),
SVC(gamma="auto"),
LinearSVC(),
]
x_test_original = self.x_test_iris.copy()
for model in scikitlearn_test_cases:
classifier = SklearnClassifier(model=model, clip_values=(0, 1))
classifier.fit(x=self.x_test_iris, y=self.y_test_iris)
# Norm=2
attack = HopSkipJump(classifier,
targeted=False,
max_iter=20,
max_eval=100,
init_eval=10)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris,
axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(
self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Accuracy of " + classifier.__class__.__name__ +
" on Iris with HopSkipJump adversarial "
"examples: %.2f%%",
(acc * 100),
)
# Norm=np.inf
attack = HopSkipJump(classifier,
targeted=False,
max_iter=20,
max_eval=100,
init_eval=10,
norm=np.Inf)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris,
axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(
self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Accuracy of " + classifier.__class__.__name__ +
" on Iris with HopSkipJump adversarial "
"examples: %.2f%%",
(acc * 100),
)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(
np.max(np.abs(x_test_original - self.x_test_iris))),
0.0,
delta=0.00001)
