def test_check_params_pt(self): ptc = get_image_classifier_pt(from_logits=True) with self.assertRaises(ValueError): _ = DefensiveDistillation(ptc, batch_size=1.0) with self.assertRaises(ValueError): _ = DefensiveDistillation(ptc, batch_size=-1) with self.assertRaises(ValueError): _ = DefensiveDistillation(ptc, nb_epochs=1.0) with self.assertRaises(ValueError): _ = DefensiveDistillation(ptc, nb_epochs=-1)
def test_2_tensorflow_iris(self): """ First test for TensorFlow. :return: """ # Create the trained classifier trained_classifier, sess = get_tabular_classifier_tf() # Create the modified classifier transformed_classifier, _ = get_tabular_classifier_tf(load_init=False, sess=sess) # Create defensive distillation transformer transformer = DefensiveDistillation(classifier=trained_classifier, batch_size=BATCH_SIZE, nb_epochs=NB_EPOCHS) # Perform the transformation with self.assertRaises(ValueError) as context: _ = transformer(x=self.x_train_iris, transformed_classifier=transformed_classifier) self.assertIn( "The input trained classifier do not produce probability outputs.", str(context.exception)) # Clean-up session if sess is not None: sess.close()
def test_6_keras_iris(self): """ Second test for Keras. :return: """ # Create the trained classifier trained_classifier = get_tabular_classifier_kr() # Create the modified classifier transformed_classifier = get_tabular_classifier_kr(load_init=False) # Create defensive distillation transformer transformer = DefensiveDistillation(classifier=trained_classifier, batch_size=BATCH_SIZE, nb_epochs=NB_EPOCHS) # Perform the transformation transformed_classifier = transformer(x=self.x_train_iris, transformed_classifier=transformed_classifier) # Compare the 2 outputs preds1 = trained_classifier.predict(x=self.x_train_iris, batch_size=BATCH_SIZE) preds2 = transformed_classifier.predict(x=self.x_train_iris, batch_size=BATCH_SIZE) preds1 = np.argmax(preds1, axis=1) preds2 = np.argmax(preds2, axis=1) acc = np.sum(preds1 == preds2) / len(preds1) self.assertGreater(acc, 0.2) ce = cross_entropy(preds1, preds2) self.assertLess(ce, 20) self.assertGreaterEqual(ce, 0)
def test_3_pytorch_classifier(self): """ Second test with the PyTorchClassifier. :return: """ self.x_train_mnist = np.reshape( self.x_train_mnist, (self.x_train_mnist.shape[0], 1, 28, 28)).astype(np.float32) # Create the trained classifier trained_classifier = get_image_classifier_pt() # Create the modified classifier transformed_classifier = get_image_classifier_pt(load_init=False) # Create defensive distillation transformer transformer = DefensiveDistillation(classifier=trained_classifier, batch_size=BATCH_SIZE, nb_epochs=NB_EPOCHS) # Perform the transformation transformed_classifier = transformer( x=self.x_train_mnist, transformed_classifier=transformed_classifier) # Compare the 2 outputs preds1 = trained_classifier.predict(x=self.x_train_mnist, batch_size=BATCH_SIZE) preds2 = transformed_classifier.predict(x=self.x_train_mnist, batch_size=BATCH_SIZE) preds1 = np.argmax(preds1, axis=1) preds2 = np.argmax(preds2, axis=1) acc = np.sum(preds1 == preds2) / len(preds1) self.assertGreater(acc, 0.5) ce = cross_entropy(preds1, preds2) self.assertLess(ce, 10) self.assertGreaterEqual(ce, 0) self.x_train_mnist = np.reshape( self.x_train_mnist, (self.x_train_mnist.shape[0], 28, 28, 1)).astype(np.float32)
def test_1_tensorflow_classifier(self): """ First test with the TensorFlowClassifier. :return: """ # Create the trained classifier trained_classifier, sess = get_image_classifier_tf() # Create the modified classifier transformed_classifier, _ = get_image_classifier_tf(load_init=False, sess=sess) # Create defensive distillation transformer transformer = DefensiveDistillation(classifier=trained_classifier, batch_size=BATCH_SIZE, nb_epochs=NB_EPOCHS) # Perform the transformation transformed_classifier = transformer( x=self.x_train_mnist, transformed_classifier=transformed_classifier) # Compare the 2 outputs preds1 = trained_classifier.predict(x=self.x_train_mnist, batch_size=BATCH_SIZE) preds2 = transformed_classifier.predict(x=self.x_train_mnist, batch_size=BATCH_SIZE) preds1 = np.argmax(preds1, axis=1) preds2 = np.argmax(preds2, axis=1) acc = np.sum(preds1 == preds2) / len(preds1) self.assertGreater(acc, 0.5) ce = cross_entropy(preds1, preds2) self.assertLess(ce, 10) self.assertGreaterEqual(ce, 0) # Clean-up session if sess is not None: sess.close()
def test_4_pytorch_iris(self): """ Third test for PyTorch. :return: """ # Create the trained classifier trained_classifier = get_tabular_classifier_pt() # Create the modified classifier transformed_classifier = get_tabular_classifier_pt(load_init=False) # Create defensive distillation transformer transformer = DefensiveDistillation(classifier=trained_classifier, batch_size=BATCH_SIZE, nb_epochs=NB_EPOCHS) # Perform the transformation with self.assertRaises(ValueError) as context: _ = transformer(x=self.x_train_iris, transformed_classifier=transformed_classifier) self.assertIn("The input trained classifier do not produce probability outputs.", str(context.exception))