def setUpIRIS(self):
(x_train, y_train), (x_test, y_test), min_, max_ = load_iris()
# Naturally IRIS has labels 0, 1, and 2. For binary classification use only classes 1 and 2.
no_zero = np.where(np.argmax(y_train, axis=1) != 0)
x_train = x_train[no_zero, :2][0]
y_train = y_train[no_zero]
no_zero = np.where(np.argmax(y_test, axis=1) != 0)
x_test = x_test[no_zero, :2][0]
y_test = y_test[no_zero]
labels = np.zeros((y_train.shape[0], 2))
labels[np.argmax(y_train, axis=1) == 2] = np.array([1, 0])
labels[np.argmax(y_train, axis=1) == 1] = np.array([0, 1])
y_train = labels
te_labels = np.zeros((y_test.shape[0], 2))
te_labels[np.argmax(y_test, axis=1) == 2] = np.array([1, 0])
te_labels[np.argmax(y_test, axis=1) == 1] = np.array([0, 1])
y_test = te_labels
n_sample = len(x_train)
order = np.random.permutation(n_sample)
x_train = x_train[order]
y_train = y_train[order].astype(np.float)
x_train = x_train[:int(.9 * n_sample)]
y_train = y_train[:int(.9 * n_sample)]
train_dups = self.find_duplicates(x_train)
x_train = x_train[np.logical_not(train_dups)]
y_train = y_train[np.logical_not(train_dups)]
test_dups = self.find_duplicates(x_test)
x_test = x_test[np.logical_not(test_dups)]
y_test = y_test[np.logical_not(test_dups)]
self.iris = (x_train, y_train), (x_test, y_test), min_, max_
def test_performance_diff(self):
from art.estimators.classification.scikitlearn import SklearnClassifier
from sklearn.svm import SVC
(x_train, y_train), (x_test, y_test), min_, max_ = load_iris()
full_model = SklearnClassifier(model=SVC(kernel="linear", gamma="auto"), clip_values=(min_, max_))
full_model.fit(x_train, y_train)
limited_model = SklearnClassifier(model=SVC(kernel="linear", gamma="auto"), clip_values=(min_, max_))
limited_model.fit(x_train[:10], y_train[:10])
self.assertEqual(
performance_diff(full_model, limited_model, x_test[:20], y_test[:20], perf_function="accuracy"), 0.35
)
self.assertEqual(performance_diff(full_model, limited_model, x_test[:20], y_test[:20]), 0.35)
diff = performance_diff(
full_model, limited_model, x_test[:20], y_test[:20], perf_function="f1", average="weighted"
)
self.assertGreater(diff, 0.43)
self.assertLess(diff, 0.44)
def first_class(true_labels, model_labels, idx=0):
return np.average(np.argmax(model_labels, axis=1) == idx)
self.assertEqual(
performance_diff(full_model, limited_model, x_test, y_test, perf_function=first_class), 1.0 / 3
)
self.assertEqual(
performance_diff(full_model, limited_model, x_test, y_test, perf_function=first_class, idx=1), -1.0 / 3
)
def test_iris(self):
(x_train, y_train), (x_test, y_test), min_, max_ = load_iris()
self.assertTrue((min_ == 0).all())
self.assertTrue((max_ == 1).all())
self.assertEqual(x_train.shape[0], y_train.shape[0])
self.assertEqual(x_test.shape[0], y_test.shape[0])
train_labels = np.argmax(y_train, axis=1)
self.assertEqual(np.setdiff1d(train_labels, np.array([0, 1, 2])).shape, (0,))
test_labels = np.argmax(y_test, axis=1)
self.assertEqual(np.setdiff1d(test_labels, np.array([0, 1, 2])).shape, (0,))
(x_train, y_train), (x_test, y_test), _, _ = load_iris(test_set=0)
self.assertEqual(x_train.shape[0], 150)
self.assertEqual(y_train.shape[0], 150)
self.assertIs(x_test, None)
self.assertIs(y_test, None)
def test_iris(self):
(x_train, y_train), (x_test, y_test), min_, max_ = load_iris()
self.assertAlmostEqual(min_, 0.012658227848101266, places=6)
self.assertEqual(max_, 1.0)
self.assertEqual(x_train.shape[0], y_train.shape[0])
self.assertEqual(x_test.shape[0], y_test.shape[0])
train_labels = np.argmax(y_train, axis=1)
self.assertEqual(
np.setdiff1d(train_labels, np.array([0, 1, 2])).shape, (0, ))
test_labels = np.argmax(y_test, axis=1)
self.assertEqual(
np.setdiff1d(test_labels, np.array([0, 1, 2])).shape, (0, ))
(x_train, y_train), (x_test, y_test), _, _ = load_iris(test_set=0)
self.assertEqual(x_train.shape[0], 150)
self.assertEqual(y_train.shape[0], 150)
self.assertIs(x_test, None)
self.assertIs(y_test, None)
