def test_7_scikitlearn(self):
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from art.estimators.classification.scikitlearn import SklearnClassifier
scikitlearn_test_cases = [
LogisticRegression(solver="lbfgs", multi_class="auto"),
SVC(gamma="auto"),
LinearSVC(),
]
x_test_original = self.x_test_iris.copy()
for model in scikitlearn_test_cases:
classifier = SklearnClassifier(model=model, clip_values=(0, 1))
classifier.fit(x=self.x_test_iris, y=self.y_test_iris)
# Test untargeted attack
attack = ProjectedGradientDescent(classifier, eps=1.0, eps_step=0.1, max_iter=5, verbose=False)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris, axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Accuracy of " + classifier.__class__.__name__ + " on Iris with PGD adversarial examples: " "%.2f%%",
(acc * 100),
)
# Test targeted attack
targets = random_targets(self.y_test_iris, nb_classes=3)
attack = ProjectedGradientDescent(
classifier, targeted=True, eps=1.0, eps_step=0.1, max_iter=5, verbose=False
)
x_test_adv = attack.generate(self.x_test_iris, **{"y": targets})
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertTrue((np.argmax(targets, axis=1) == preds_adv).any())
acc = np.sum(preds_adv == np.argmax(targets, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Success rate of " + classifier.__class__.__name__ + " on targeted PGD on Iris: %.2f%%", (acc * 100)
)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - self.x_test_iris))), 0.0, delta=0.00001)
def test_GradientBoosting(self):
model = GradientBoostingClassifier(n_estimators=4, max_depth=6)
model.fit(self.x_train, np.argmax(self.y_train, axis=1))
classifier = SklearnClassifier(model=model)
rt = RobustnessVerificationTreeModelsCliqueMethod(
classifier=classifier)
average_bound, verified_error = rt.verify(x=self.x_test,
y=self.y_test,
eps_init=0.3,
nb_search_steps=10,
max_clique=2,
max_level=2)
self.assertAlmostEqual(average_bound,
0.008982421874999,
delta=0.00000001)
self.assertEqual(verified_error, 1.0)
def test_scikitlearn(self):
clf = DecisionTreeClassifier()
x_original = self.X.copy()
clf.fit(self.X, self.y)
clf_art = SklearnClassifier(clf)
attack = DecisionTreeAttack(clf_art, verbose=False)
adv = attack.generate(self.X[:25])
# all crafting should succeed
self.assertTrue(
np.sum(clf.predict(adv) == clf.predict(self.X[:25])) == 0)
targets = np.array([
1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8,
8, 8, 9
])
adv = attack.generate(self.X[:25], targets)
# all targeted crafting should succeed as well
self.assertTrue(np.sum(clf.predict(adv) == targets) == 25.0)
# Check that X has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_original - self.X))),
0.0,
delta=0.00001)
def test_6_scikitlearn(self):
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier
from sklearn.ensemble import AdaBoostClassifier, BaggingClassifier, ExtraTreesClassifier
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from art.estimators.classification.scikitlearn import SklearnClassifier
scikitlearn_test_cases = [
DecisionTreeClassifier(),
ExtraTreeClassifier(),
AdaBoostClassifier(),
BaggingClassifier(),
ExtraTreesClassifier(n_estimators=10),
GradientBoostingClassifier(n_estimators=10),
RandomForestClassifier(n_estimators=10),
LogisticRegression(solver="lbfgs", multi_class="auto"),
SVC(gamma="auto"),
LinearSVC(),
]
x_test_original = self.x_test_iris.copy()
for model in scikitlearn_test_cases:
classifier = SklearnClassifier(model=model, clip_values=(0, 1))
classifier.fit(x=self.x_test_iris, y=self.y_test_iris)
# Norm=2
attack = HopSkipJump(classifier,
targeted=False,
max_iter=20,
max_eval=100,
init_eval=10)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris,
axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(
self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Accuracy of " + classifier.__class__.__name__ +
" on Iris with HopSkipJump adversarial "
"examples: %.2f%%",
(acc * 100),
)
# Norm=np.inf
attack = HopSkipJump(classifier,
targeted=False,
max_iter=20,
max_eval=100,
init_eval=10,
norm=np.Inf)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertTrue((x_test_adv <= 1).all())
self.assertTrue((x_test_adv >= 0).all())
preds_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris,
axis=1) == preds_adv).all())
acc = np.sum(preds_adv == np.argmax(
self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Accuracy of " + classifier.__class__.__name__ +
" on Iris with HopSkipJump adversarial "
"examples: %.2f%%",
(acc * 100),
)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(
np.max(np.abs(x_test_original - self.x_test_iris))),
0.0,
delta=0.00001)
def test_7_scikitlearn(self):
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from art.estimators.classification.scikitlearn import SklearnClassifier
scikitlearn_test_cases = [
LogisticRegression(solver="lbfgs", multi_class="auto"),
SVC(gamma="auto"),
LinearSVC(),
]
x_test_original = self.x_test_iris.copy()
for model in scikitlearn_test_cases:
classifier = SklearnClassifier(model=model, clip_values=(0, 1))
classifier.fit(x=self.x_test_iris, y=self.y_test_iris)
# Test untargeted attack
attack = ElasticNet(classifier,
targeted=False,
max_iter=2,
verbose=False)
x_test_adv = attack.generate(self.x_test_iris)
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
predictions_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertFalse((np.argmax(self.y_test_iris,
axis=1) == predictions_adv).all())
accuracy = 1.0 - np.sum(predictions_adv == np.argmax(
self.y_test_iris, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"EAD success rate of " + classifier.__class__.__name__ +
" on Iris: %.2f%%", (accuracy * 100))
# Test targeted attack
targets = random_targets(self.y_test_iris, nb_classes=3)
attack = ElasticNet(classifier,
targeted=True,
max_iter=2,
verbose=False)
x_test_adv = attack.generate(self.x_test_iris, **{"y": targets})
self.assertFalse((self.x_test_iris == x_test_adv).all())
self.assertLessEqual(np.amax(x_test_adv), 1.0)
self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
predictions_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
self.assertTrue((np.argmax(targets,
axis=1) == predictions_adv).any())
accuracy = np.sum(predictions_adv == np.argmax(
targets, axis=1)) / self.y_test_iris.shape[0]
logger.info(
"Targeted EAD success rate of " +
classifier.__class__.__name__ + " on Iris: %.2f%%",
(accuracy * 100))
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(
np.max(np.abs(x_test_original - self.x_test_iris))),
0.0,
delta=0.00001)
def setUpClass(cls):
master_seed(seed=301)
(x_train, y_train), (x_test, y_test), min_, max_ = load_mnist()
y_train = np.argmax(y_train, axis=1)
y_test = np.argmax(y_test, axis=1)
zero_or_four = np.logical_or(y_train == 4, y_train == 0, y_train == 9)
x_train = x_train[zero_or_four]
y_train = y_train[zero_or_four]
tr_labels = np.zeros((y_train.shape[0], 2))
tr_labels[y_train == 0] = np.array([1, 0])
tr_labels[y_train == 4] = np.array([0, 1])
y_train = tr_labels
zero_or_four = np.logical_or(y_test == 4, y_test == 0)
x_test = x_test[zero_or_four]
y_test = y_test[zero_or_four]
te_labels = np.zeros((y_test.shape[0], 2))
te_labels[y_test == 0] = np.array([1, 0])
te_labels[y_test == 4] = np.array([0, 1])
y_test = te_labels
n_samples_train = x_train.shape[0]
n_features_train = x_train.shape[1] * x_train.shape[2] * x_train.shape[3]
n_samples_test = x_test.shape[0]
n_features_test = x_test.shape[1] * x_test.shape[2] * x_test.shape[3]
x_train = x_train.reshape(n_samples_train, n_features_train)
x_test = x_test.reshape(n_samples_test, n_features_test)
x_train = x_train[:NB_TRAIN]
y_train = y_train[:NB_TRAIN]
trusted_data = x_test[:NB_TRUSTED]
trusted_labels = y_test[:NB_TRUSTED]
x_test = x_test[NB_TRUSTED:]
y_test = y_test[NB_TRUSTED:]
valid_data = x_test[:NB_VALID]
valid_labels = y_test[:NB_VALID]
x_test = x_test[NB_VALID:]
y_test = y_test[NB_VALID:]
clean_prov = np.random.randint(NB_DEVICES - 1, size=x_train.shape[0])
p_train = np.eye(NB_DEVICES)[clean_prov]
no_defense = ScikitlearnSVC(model=SVC(kernel=kernel, gamma="auto"), clip_values=(min_, max_))
no_defense.fit(x=x_train, y=y_train)
poison_points = np.random.randint(no_defense._model.support_vectors_.shape[0], size=NB_POISON)
all_poison_init = np.copy(no_defense._model.support_vectors_[poison_points])
poison_labels = np.array([1, 1]) - no_defense.predict(all_poison_init)
svm_attack = PoisoningAttackSVM(
classifier=no_defense,
x_train=x_train,
y_train=y_train,
step=0.1,
eps=1.0,
x_val=valid_data,
y_val=valid_labels,
)
poisoned_data, _ = svm_attack.poison(all_poison_init, y=poison_labels)
# Stack on poison to data and add provenance of bad actor
all_data = np.vstack([x_train, poisoned_data])
all_labels = np.vstack([y_train, poison_labels])
poison_prov = np.zeros((NB_POISON, NB_DEVICES))
poison_prov[:, NB_DEVICES - 1] = 1
all_p = np.vstack([p_train, poison_prov])
model = SVC(kernel=kernel, gamma="auto")
cls.mnist = (
(all_data, all_labels, all_p),
(x_test, y_test),
(trusted_data, trusted_labels),
(valid_data, valid_labels),
(min_, max_),
)
cls.classifier = SklearnClassifier(model=model, clip_values=(min_, max_))
cls.classifier.fit(all_data, all_labels)
cls.defence_trust = ProvenanceDefense(
cls.classifier, all_data, all_labels, all_p, x_val=trusted_data, y_val=trusted_labels, eps=0.1,
)
cls.defence_no_trust = ProvenanceDefense(cls.classifier, all_data, all_labels, all_p, eps=0.1)
def test_type(self):
self.assertIsInstance(
self.classifier, type(SklearnClassifier(model=self.sklearn_model)))
with self.assertRaises(TypeError):
ScikitlearnExtraTreeClassifier(model="sklearn_model")
def test_type(self):
self.assertIsInstance(
self.classifier, type(SklearnClassifier(model=self.sklearn_model)))
with self.assertRaises(TypeError):
ScikitlearnLogisticRegression(model="sklearn_model")
def test_type(self):
self.assertIsInstance(
self.classifier, type(SklearnClassifier(model=self.sklearn_model)))
with self.assertRaises(TypeError):
ScikitlearnGradientBoostingClassifier(model="sklearn_model")
def test_type(self):
self.assertIsInstance(self.classifier, type(SklearnClassifier(model=self.sklearn_model)))
def test_SVC_kernels(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
# Get MNIST
(x_train, y_train), (x_test, y_test), min_, max_ = self.iris
x_test_original = x_test.copy()
# Build Scikitlearn Classifier
clip_values = (min_, max_)
for kernel in ["linear", "poly", "rbf"]:
clean = SklearnClassifier(model=SVC(kernel=kernel, gamma="auto"),
clip_values=clip_values)
clean.fit(x_train, y_train)
poison = SklearnClassifier(model=SVC(kernel=kernel, gamma="auto"),
clip_values=clip_values)
poison.fit(x_train, y_train)
attack = PoisoningAttackSVM(poison, 0.01, 1.0, x_train, y_train,
x_test, y_test, 100)
attack_y = np.array([1, 1]) - y_train[0]
attack_point, _ = attack.poison(np.array([x_train[0]]),
y=np.array([attack_y]))
poison.fit(
x=np.vstack([x_train, attack_point]),
y=np.vstack([
y_train,
np.array([1, 1]) - np.copy(y_train[0].reshape((1, 2)))
]),
)
acc = np.average(np.all(clean.predict(x_test) == y_test,
axis=1)) * 100
poison_acc = np.average(
np.all(poison.predict(x_test) == y_test, axis=1)) * 100
logger.info("Clean Accuracy {}%".format(acc))
logger.info("Poison Accuracy {}%".format(poison_acc))
self.assertGreaterEqual(acc, poison_acc)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(
np.max(np.abs(x_test_original - x_test))),
0.0,
delta=0.00001)
def test_linearSVC(self):
"""
Test using a attack on LinearSVC
"""
(x_train, y_train), (x_test, y_test), min_, max_ = self.iris
x_test_original = x_test.copy()
# Build Scikitlearn Classifier
clip_values = (min_, max_)
clean = SklearnClassifier(model=LinearSVC(), clip_values=clip_values)
clean.fit(x_train, y_train)
poison = SklearnClassifier(model=LinearSVC(), clip_values=clip_values)
poison.fit(x_train, y_train)
attack = PoisoningAttackSVM(poison, 0.01, 1.0, x_train, y_train,
x_test, y_test, 100)
attack_y = np.array([1, 1]) - y_train[0]
attack_point, _ = attack.poison(np.array([x_train[0]]),
y=np.array([attack_y]))
poison.fit(x=np.vstack([x_train, attack_point]),
y=np.vstack([y_train,
np.copy(y_train[0].reshape((1, 2)))]))
acc = np.average(np.all(clean.predict(x_test) == y_test, axis=1)) * 100
poison_acc = np.average(
np.all(poison.predict(x_test) == y_test, axis=1)) * 100
logger.info("Clean Accuracy {}%".format(acc))
logger.info("Poison Accuracy {}%".format(poison_acc))
self.assertGreaterEqual(acc, poison_acc)
# Check that x_test has not been modified by attack and classifier
self.assertAlmostEqual(float(np.max(np.abs(x_test_original - x_test))),
0.0,
delta=0.00001)
