def test_get_probs(self): import tensorflow as tf model = KerasModelWrapper(self.model) x = tf.placeholder(tf.float32, shape=(None, 100)) preds = model.get_probs(x) x_val = np.random.rand(2, 100) tf.global_variables_initializer().run(session=self.sess) p_val = self.sess.run(preds, feed_dict={x: x_val}) self.assertTrue(np.allclose(np.sum(p_val, axis=1), 1, atol=1e-6)) self.assertTrue(np.all(p_val>=0)) self.assertTrue(np.all(p_val<=1))
def test_get_logits(self): import tensorflow as tf model = KerasModelWrapper(self.model) x = tf.placeholder(tf.float32, shape=(None, 100)) preds = model.get_probs(x) logits = model.get_logits(x) x_val = np.random.rand(2, 100) tf.global_variables_initializer().run(session=self.sess) p_val, logits = self.sess.run([preds, logits], feed_dict={x: x_val}) p_gt = np.exp(logits)/np.sum(np.exp(logits), axis=1, keepdims=True) self.assertTrue(np.allclose(p_val, p_gt, atol=1e-6))
def test_fprop(self): import tensorflow as tf model = KerasModelWrapper(self.model) x = tf.placeholder(tf.float32, shape=(None, 100)) out_dict = model.fprop(x) self.assertEqual(set(out_dict.keys()), set(['l1', 'l2', 'softmax'])) # Test the dimension of the hidden represetation self.assertEqual(int(out_dict['l1'].shape[1]), 20) self.assertEqual(int(out_dict['l2'].shape[1]), 10) # Test the caching x2 = tf.placeholder(tf.float32, shape=(None, 100)) out_dict2 = model.fprop(x2) self.assertEqual(set(out_dict2.keys()), set(['l1', 'l2', 'softmax'])) self.assertEqual(int(out_dict2['l1'].shape[1]), 20)
def prep_bbox(sess, x, y, X_train, Y_train, X_test, Y_test, nb_epochs, batch_size, learning_rate, rng, nb_classes=10, img_rows=28, img_cols=28, nchannels=1): """ Define and train a model that simulates the "remote" black-box oracle described in the original paper. :param sess: the TF session :param x: the input placeholder for MNIST :param y: the ouput placeholder for MNIST :param X_train: the training data for the oracle :param Y_train: the training labels for the oracle :param X_test: the testing data for the oracle :param Y_test: the testing labels for the oracle :param nb_epochs: number of epochs to train model :param batch_size: size of training batches :param learning_rate: learning rate for training :param rng: numpy.random.RandomState :return: """ # Define Keras-based TF model graph (for the black-box model) nb_filters = 64 model = cnn_model(nb_filters=nb_filters, nb_classes=nb_classes) # Wrap the model in KerasModelWrapper model = KerasModelWrapper(model, nb_classes) loss = LossCrossEntropy(model, smoothing=0.1) predictions = model.get_logits(x) print("Defined TensorFlow model graph.") # Train an MNIST model train_params = { 'nb_epochs': nb_epochs, 'batch_size': batch_size, 'learning_rate': learning_rate } train(sess, loss, x, y, X_train, Y_train, args=train_params, rng=rng) # Print out the accuracy on legitimate data eval_params = {'batch_size': batch_size} accuracy = model_eval(sess, x, y, predictions, X_test, Y_test, args=eval_params) print('Test accuracy of black-box on legitimate test ' 'examples: ' + str(accuracy)) return model, predictions, accuracy
def lbfgs(X, which): wrapped = LBFGS(KerasModelWrapper(which.model), sess=session) X = X.copy() for i in tqdm(range(0, len(X), CHILD_BATCH_SIZE), desc=f'batch: ', leave=False): tensor = tf.convert_to_tensor(X[i:i + CHILD_BATCH_SIZE]) tensor = wrapped.generate(tensor, eps=0.1) X[i:i + CHILD_BATCH_SIZE] = session.run(tensor) return X
def basic_iter_init(model): """ @brief: Initialize the Basic Iterative module with Keras model @param: Keras model (TensorFlow) @return: Intialized Basic Iterative module """ biter_wrapper = KerasModelWrapper(model) biter_session = keras.backend.get_session() return clhan.BasicIterativeMethod(biter_wrapper, biter_session)
def attack(model, x_input, input_img): wrap = KerasModelWrapper(model) jsma = SaliencyMapMethod(wrap, sess=sess) jsma_params = { 'theta': 0.1, 'gamma': 0.1, 'clip_min': -1., 'clip_max': 1., 'y_target': None } adv_x = jsma.generate_np(input_img, **jsma_params) return adv_x
def run(vgg=False, resnet=False, net_in_net=False, densenet=False, attack_name=None, train=True, holdout=150): start = timeit.default_timer() keras.layers.core.K.set_learning_phase(0) sess = tf.Session() keras.backend.set_session(sess) model_wrapper = get_model_wrapper(vgg, resnet, net_in_net, densenet) if model_wrapper is None: Exception("No model provided") model = model_wrapper.model x_train, x_test, y_train, y_test = prepare_cifar_data(vgg=vgg, resnet=resnet, net_in_net=net_in_net, densenet=densenet, train=train) # Initialize substitute training set reserved for adversary x_sub = x_test[:holdout] y_sub = np.argmax(y_test[:holdout], axis=1) # Redefine test set as remaining samples unavailable to adversaries x_test = x_test[holdout:] y_test = y_test[holdout:] x = tf.placeholder(tf.float32, shape=(None, 32, 32, 3)) y = tf.placeholder(tf.float32, shape=(None, 10)) bbox_preds = model(x) print("Training the substitute model.") train_sub_out = train_sub(sess, x, y, bbox_preds, x_sub, y_sub) model_sub, preds_sub = train_sub_out # Evaluate the substitute model on clean test examples eval_params = {'batch_size': 128} acc = model_eval(sess, x, y, preds_sub, x_test, y_test, args=eval_params) print('Test accuracy of substitute on legitimate examples ' + str(acc)) wrap = KerasModelWrapper(model_sub) attack, params, stop_gradient = get_adversarial_attack_and_params(attack_name, wrap, sess) params = {"y_target": y, "batch_size": 1} adv_x = attack.generate(x, **params) if params else attack.generate(x) if stop_gradient: # Consider the attack to be constant adv_x = tf.stop_gradient(adv_x) # Evaluate the accuracy of the "black-box" model on adversarial examples accuracy = model_eval(sess, x, y, model(adv_x), x_test, y_test, args=eval_params) print('Test accuracy of oracle on adversarial examples generated ' 'using the substitute: ' + str(accuracy)) stop = timeit.default_timer() print(str(stop - start))
def fgsm_init(model): """ @brief: Initialize the FGSM module with the Keras Model @param: Tensor Flow Model @return: Intialized FGSM module """ fgsm_wrapper = KerasModelWrapper(model) fgsm_session = keras.backend.get_session() return clhan.FastGradientMethod(fgsm_wrapper, fgsm_session)
class VGG16(Model): __metaclass__ = ABCMeta def __init__(self): Model.__init__(self) from keras.applications.vgg16 import VGG16, preprocess_input, decode_predictions self.keras_model = VGG16(weights='imagenet') self.model = KerasModelWrapper(self.keras_model) self.preprocess_input = preprocess_input self.decode_predictions = decode_predictions def get_logits(self, x): return self.model.get_logits(self.preprocess_input(x)) def get_probs(self, x): return self.model.get_probs(self.preprocess_input(x)) def get_layer(self, x, layer): output = self.model.fprop(self.preprocess_input(x)) try: requested = output[layer] except KeyError: raise NoSuchLayerError() return requested def get_layer_names(self): """ :return: Names of all the layers kept by Keras """ layer_names = [x.name for x in self.keras_model.layers] return layer_names def predict(self, x, preprocess=False): if preprocess: return self.keras_model.predict(self.preprocess_input(x)) else: return self.keras_model.predict(x)
def fgsm(X, which, prob, magn): wrapped = FastGradientMethod(KerasModelWrapper(which.model), sess=session) X = X.copy() idx = np.random.uniform(size=len(X)) idx = np.where(idx < prob)[0] for i in tqdm(range(0, len(idx), CHILD_BATCH_SIZE), desc=f'batch: ', leave=False): tensor = tf.convert_to_tensor(X[idx[i:i + CHILD_BATCH_SIZE]]) init = tf.global_variables_initializer() session.run(init) tensor = wrapped.generate(tensor, eps=0.1 * magn) X[idx[i:i + CHILD_BATCH_SIZE]] = session.run(tensor) return X
def __init__(self, sess, test_batch_size, type = 1,use_softmax = True, x = None,y = None, is_training=None,\ keep_prob=None,load_existing = False, model_name = 'modelA', loss = 'cw'): self.x = x self.y = y self.sess = sess self.is_training = is_training self.keep_prob = keep_prob self.test_batch_size = test_batch_size if load_existing: save_dir = 'CIFAR10_models/Normal_simple_models' # TODO: put your own ROOT directory of simple cifar10 models filepath = os.path.join(save_dir, model_name + '.h5') model = load_model(filepath) self.model = model model = KerasModelWrapper(model) self.predictions = model.get_logits(self.x) else: model, preds = model_cifar10(input_ph=x, type=type) self.model = model self.predictions = preds self.probs = tf.nn.softmax(logits=self.predictions) self.eval_preds = tf.argmax(self.predictions, 1) self.y_target = tf.placeholder(tf.int64, shape=None) # tensor.shape (?,) self.eval_percent_adv = tf.equal( self.eval_preds, self.y_target) # one-to-one comparison if loss == 'cw': self.target_logits = tf.reduce_sum(self.y * self.predictions, 1) self.other_logits = tf.reduce_max( (1 - self.y) * self.predictions - (self.y * 10000), 1) self.loss = self.other_logits - self.target_logits elif loss == 'xent': self.loss = tf.nn.softmax_cross_entropy_with_logits( labels=self.y, logits=self.predictions) else: raise NotImplementedError
def df(X, which, prob, magn): wrapped = DeepFool(KerasModelWrapper(which.model), sess=session) X = X.copy() idx = np.random.uniform(size=len(X)) idx = np.where(idx < prob)[0] for i in tqdm(range(0, len(idx), CHILD_BATCH_SIZE), desc=f'batch: ', leave=False): tensor = tf.convert_to_tensor(X[idx[i:i + CHILD_BATCH_SIZE]]) init = tf.global_variables_initializer() session.run(init) tensor = wrapped.generate(tensor, clip_min=0., clip_max=magn * 0.3 + 0.3) X[idx[i:i + CHILD_BATCH_SIZE]] = session.run(tensor) return X
def test_get_layer_names(self): model = KerasModelWrapper(self.model) layer_names = model.get_layer_names() self.assertEqual(layer_names, ['l1', 'l2', 'softmax'])
def test_logit_layer_name_is_logits(self): model = KerasModelWrapper(self.model) logits_name = model._get_logits_name() self.assertEqual(logits_name, 'l2')
def test_softmax_layer_name_is_softmax(self): model = KerasModelWrapper(self.model) softmax_name = model._get_softmax_name() self.assertEqual(softmax_name, 'softmax')