def test_check_params_pt(self):
ptc = get_image_classifier_pt(from_logits=True)
with self.assertRaises(ValueError):
_ = DefensiveDistillation(ptc, batch_size=1.0)
with self.assertRaises(ValueError):
_ = DefensiveDistillation(ptc, batch_size=-1)
with self.assertRaises(ValueError):
_ = DefensiveDistillation(ptc, nb_epochs=1.0)
with self.assertRaises(ValueError):
_ = DefensiveDistillation(ptc, nb_epochs=-1)
def test_2_tensorflow_iris(self):
"""
First test for TensorFlow.
:return:
"""
# Create the trained classifier
trained_classifier, sess = get_tabular_classifier_tf()
# Create the modified classifier
transformed_classifier, _ = get_tabular_classifier_tf(load_init=False,
sess=sess)
# Create defensive distillation transformer
transformer = DefensiveDistillation(classifier=trained_classifier,
batch_size=BATCH_SIZE,
nb_epochs=NB_EPOCHS)
# Perform the transformation
with self.assertRaises(ValueError) as context:
_ = transformer(x=self.x_train_iris,
transformed_classifier=transformed_classifier)
self.assertIn(
"The input trained classifier do not produce probability outputs.",
str(context.exception))
# Clean-up session
if sess is not None:
sess.close()
def test_6_keras_iris(self):
"""
Second test for Keras.
:return:
"""
# Create the trained classifier
trained_classifier = get_tabular_classifier_kr()
# Create the modified classifier
transformed_classifier = get_tabular_classifier_kr(load_init=False)
# Create defensive distillation transformer
transformer = DefensiveDistillation(classifier=trained_classifier, batch_size=BATCH_SIZE, nb_epochs=NB_EPOCHS)
# Perform the transformation
transformed_classifier = transformer(x=self.x_train_iris, transformed_classifier=transformed_classifier)
# Compare the 2 outputs
preds1 = trained_classifier.predict(x=self.x_train_iris, batch_size=BATCH_SIZE)
preds2 = transformed_classifier.predict(x=self.x_train_iris, batch_size=BATCH_SIZE)
preds1 = np.argmax(preds1, axis=1)
preds2 = np.argmax(preds2, axis=1)
acc = np.sum(preds1 == preds2) / len(preds1)
self.assertGreater(acc, 0.2)
ce = cross_entropy(preds1, preds2)
self.assertLess(ce, 20)
self.assertGreaterEqual(ce, 0)
def test_3_pytorch_classifier(self):
"""
Second test with the PyTorchClassifier.
:return:
"""
self.x_train_mnist = np.reshape(
self.x_train_mnist,
(self.x_train_mnist.shape[0], 1, 28, 28)).astype(np.float32)
# Create the trained classifier
trained_classifier = get_image_classifier_pt()
# Create the modified classifier
transformed_classifier = get_image_classifier_pt(load_init=False)
# Create defensive distillation transformer
transformer = DefensiveDistillation(classifier=trained_classifier,
batch_size=BATCH_SIZE,
nb_epochs=NB_EPOCHS)
# Perform the transformation
transformed_classifier = transformer(
x=self.x_train_mnist,
transformed_classifier=transformed_classifier)
# Compare the 2 outputs
preds1 = trained_classifier.predict(x=self.x_train_mnist,
batch_size=BATCH_SIZE)
preds2 = transformed_classifier.predict(x=self.x_train_mnist,
batch_size=BATCH_SIZE)
preds1 = np.argmax(preds1, axis=1)
preds2 = np.argmax(preds2, axis=1)
acc = np.sum(preds1 == preds2) / len(preds1)
self.assertGreater(acc, 0.5)
ce = cross_entropy(preds1, preds2)
self.assertLess(ce, 10)
self.assertGreaterEqual(ce, 0)
self.x_train_mnist = np.reshape(
self.x_train_mnist,
(self.x_train_mnist.shape[0], 28, 28, 1)).astype(np.float32)
def test_1_tensorflow_classifier(self):
"""
First test with the TensorFlowClassifier.
:return:
"""
# Create the trained classifier
trained_classifier, sess = get_image_classifier_tf()
# Create the modified classifier
transformed_classifier, _ = get_image_classifier_tf(load_init=False,
sess=sess)
# Create defensive distillation transformer
transformer = DefensiveDistillation(classifier=trained_classifier,
batch_size=BATCH_SIZE,
nb_epochs=NB_EPOCHS)
# Perform the transformation
transformed_classifier = transformer(
x=self.x_train_mnist,
transformed_classifier=transformed_classifier)
# Compare the 2 outputs
preds1 = trained_classifier.predict(x=self.x_train_mnist,
batch_size=BATCH_SIZE)
preds2 = transformed_classifier.predict(x=self.x_train_mnist,
batch_size=BATCH_SIZE)
preds1 = np.argmax(preds1, axis=1)
preds2 = np.argmax(preds2, axis=1)
acc = np.sum(preds1 == preds2) / len(preds1)
self.assertGreater(acc, 0.5)
ce = cross_entropy(preds1, preds2)
self.assertLess(ce, 10)
self.assertGreaterEqual(ce, 0)
# Clean-up session
if sess is not None:
sess.close()
def test_4_pytorch_iris(self):
"""
Third test for PyTorch.
:return:
"""
# Create the trained classifier
trained_classifier = get_tabular_classifier_pt()
# Create the modified classifier
transformed_classifier = get_tabular_classifier_pt(load_init=False)
# Create defensive distillation transformer
transformer = DefensiveDistillation(classifier=trained_classifier, batch_size=BATCH_SIZE, nb_epochs=NB_EPOCHS)
# Perform the transformation
with self.assertRaises(ValueError) as context:
_ = transformer(x=self.x_train_iris, transformed_classifier=transformed_classifier)
self.assertIn("The input trained classifier do not produce probability outputs.", str(context.exception))
