def calculate_accuracy_adv_spatial_smoothing(sess,
                                             x,
                                             y,
                                             predictions,
                                             y_test,
                                             X_test,
                                             X_test_adv,
                                             eval_param=None):

    print("\n===Calculating the accuracy with feature squeezing...")
    for width in range(1, 11):
        # height = width
        for height in range(1, 11):
            X_squeezed = median_filter_np(X_test, width, height)
            X_adv_squeezed = median_filter_np(X_test_adv, width, height)

            accuracy_leg, _ = model_test_eval(sess,
                                              x,
                                              y,
                                              predictions,
                                              X_squeezed,
                                              y_test,
                                              args=eval_param)
            accuracy_mal, _ = model_test_eval(sess,
                                              x,
                                              y,
                                              predictions,
                                              X_adv_squeezed,
                                              y_test,
                                              args=eval_param)

            print(
                "Width: %2d, Height: %2d, Accuracy_legitimate: %.2f, Accuracy_malicious: %.2f"
                % (width, height, accuracy_leg, accuracy_mal))
def calculate_accuracy_adv_feature_squeezing(sess,
                                             x,
                                             y,
                                             logits,
                                             logits_bin,
                                             X_adv,
                                             y_test,
                                             eval_params=None):

    accuracy, pred = model_test_eval(sess,
                                     x,
                                     y,
                                     logits,
                                     X_adv,
                                     y_test,
                                     args=eval_params)
    print('Test accuracy of white-box on adversarial '
          'examples: {:.3f}'.format(accuracy))

    accuracy, pred = model_test_eval(sess,
                                     x,
                                     y,
                                     logits_bin,
                                     X_adv,
                                     y_test,
                                     args=eval_params)
    print('Test accuracy of white-box on feature squeezing adversarial '
          'examples: {:.3f}'.format(accuracy))
def eval(mu, sigma, add_dropout, batch_size):
    """
    Evaluate a GTSRB model
    :param mu: A 0-D Tensor or Python value of type `dtype`. The mean of the
    truncated normal distribution.
    :param sigma: A 0-D Tensor or Python value of type `dtype`. The standard deviation
    of the truncated normal distribution.
    :param add_dropout: if or not use dropout.
    :param batch_size: size of training batches
    :return: a dictionary with:
             * model training accuracy and loss on training data
             * model validating accuracy and loss on validation data
             * accuracy on test dataset by class lebel
    """

    np.random.seed(0)
    assert keras.backend.backend() == "tensorflow"

    # # Create TF session and set as Keras backend session
    sess = tf.Session()
    keras.backend.set_session(sess)

    # Get GTSRB data
    _, _, X_test, y_test = load_data(train_data_dir, test_data_dir)

    # One-Hot Encode
    label_binarizer = LabelBinarizer()
    y_test = label_binarizer.fit_transform(y_test)
    print(y_test.shape)

    # Define input and output TF placeholders
    x = tf.placeholder(tf.float32, (None, 32, 32, 3))
    y = tf.placeholder(tf.int32, (None))

    one_hot_y = tf.one_hot(y, 43)

    # model = LeNet(model_dir)
    # logits, _ = model(x, add_dropout)
    model = LeNet(model_dir, mu, sigma)
    logits = model(x)

    print('loading LeNet ...')
    # Parameters required by evaluating
    eval_params = {'batch_size': batch_size}
    accuracy, pred = model_test_eval(sess,
                                     x,
                                     one_hot_y,
                                     logits,
                                     X_test,
                                     y_test,
                                     args=eval_params)
    print('Test accuracy of LeNet on legitimate test '
          'examples: {:.3f}'.format(accuracy))
Exemple #4
0
def eval(mu, sigma, add_dropout, batch_size):
    """
    Evaluate a GTSRB model
    :param mu: A 0-D Tensor or Python value of type `dtype`. The mean of the
    truncated normal distribution.
    :param sigma: A 0-D Tensor or Python value of type `dtype`. The standard deviation
    of the truncated normal distribution.
    :param add_dropout: if or not use dropout.
    :param batch_size: size of training batches
    :return: a dictionary with:
             * model training accuracy and loss on training data
             * model validating accuracy and loss on validation data
             * accuracy on test dataset by class lebel
    """

    np.random.seed(0)
    assert keras.backend.backend() == "tensorflow"

    # # Create TF session and set as Keras backend session
    sess = tf.Session()
    keras.backend.set_session(sess)

    # Get MNIST data
    mnist = input_data.read_data_sets('/tmp/', one_hot=True, reshape=False)
    X_train, y_train, X_test, y_test = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels

    # One-Hot Encode
    # label_binarizer = LabelBinarizer()
    # y_test = label_binarizer.fit_transform(y_test)
    # print(y_test.shape)

    # Define input and output TF placeholders
    x = tf.placeholder(tf.float32, (None, 28, 28, 1))
    y = tf.placeholder(tf.float32, (None, 10))

    # one_hot_y = tf.one_hot(y, 43)

    model = MNIST(model_dir)
    logits, _ = model(x, add_dropout)

    print('loading MNIST ...')
    # Parameters required by evaluating
    eval_params = {'batch_size': batch_size}
    accuracy, pred = model_test_eval(sess,
                                     x,
                                     y,
                                     logits,
                                     X_test,
                                     y_test,
                                     args=eval_params)
    print('Test accuracy of MNIST on legitimate test '
          'examples: {:.3f}'.format(accuracy))
def gtsrb_whitebox(attack, mu, sigma, batch_size, eps, steps, kappa, alpha):
    """
    GTSRB tutorial for the white-box attack.
    :param attack: attack type(FGSM, I-FGSM, Carlini)
    :param mu: A 0-D Tensor or Python value of type `dtype`. The mean of the
    truncated normal distribution.
    :param sigma: A 0-D Tensor or Python value of type `dtype`. The standard deviation
    of the truncated normal distribution.
    :param dropout: dropout value
    :param add_dropout: if or not dropout.
    :param batch_size: size of training batches
    :param eps: the epsilon (input variation parameter)
    :return: a dictionary with:
             * white-box model accuracy on test set
             * white-box model accuracy on adversarial examples transferred
               from the substitute model
    """

    np.random.seed(0)
    assert keras.backend.backend() == "tensorflow"

    # # Create TF session and set as Keras backend session
    sess = tf.Session()
    keras.backend.set_session(sess)

    # Get GTSRB data
    _, _, X_test, y_test = load_data(train_data_dir, test_data_dir)

    # One-Hot Encode
    label_binarizer = LabelBinarizer()
    y_test = label_binarizer.fit_transform(y_test)

    # Define input and output TF placeholders
    x = tf.placeholder(tf.float32, (None, 32, 32, 3))
    y = tf.placeholder(tf.int32, (None))

    one_hot_y = tf.one_hot(y, 43)

    print('Preparing the white-box model LeNet.')
    model = LeNet(model_dir, mu, sigma)
    logits = model(x)

    # Parameters required by evaluating
    eval_params = {'batch_size': batch_size}

    accuracy, pred = model_test_eval(sess,
                                     x,
                                     one_hot_y,
                                     logits,
                                     X_test,
                                     y_test,
                                     args=eval_params)
    print('Test accuracy of white-box on legitimate test '
          'examples: {:.3f}'.format(accuracy))

    model = AlexNet(model_dir, mu, sigma)
    logits = model(x)
    # take the random step in the RAND+FGSM
    if attack == "rand_fgsm":
        X_test = np.clip(
            X_test + alpha * np.sign(np.random.randn(*X_test.shape)), 0.0, 1.0)
        eps -= alpha

    grad = generate_gradient(x, logits, one_hot_y, loss='training')
    # FGSM and RAND_FGSM one-shot attack
    if attack in ["fgsm", "rand_fgsm"]:
        adv_x = fgsm(x, grad, eps, clipping=True)
    if attack == 'ifgsm':
        adv_x = iteration_fgsm(model,
                               x,
                               one_hot_y,
                               steps,
                               eps,
                               loss='training')

    if attack == 'Carlini':
        X_test = X_test[0:128]
        Y_test = y_test[0:128]

        # cli = CarliniL2(sess, model, targeted=False, max_iterations=1000, confidence=0, batch_size=1)
        cli = CarliniLi(sess, model, targeted=False, max_iterations=1000)
        X_adv = cli.attack(X_test, Y_test)

        r = np.clip(X_adv - X_test, -eps, eps)
        X_adv = X_test + r

        accuracy, pred = model_test_eval(sess,
                                         x,
                                         one_hot_y,
                                         logits,
                                         X_adv,
                                         y_test,
                                         args=eval_params)
        print('Test accuracy of white-box on adversarial examples crafted by '
              'Carlini Attack: {:.3f}'.format(accuracy))
        display_leg_adv_sample(X_test, X_adv)
        return

    # compute the adversarial examples and evaluate
    X_adv = batch_eval(sess, [x, y], [adv_x], [X_test, y_test],
                       args=eval_params)[0]
    accuracy, pred = model_test_eval(sess,
                                     x,
                                     one_hot_y,
                                     logits,
                                     X_adv,
                                     y_test,
                                     args=eval_params)
    print('Test accuracy of white-box on adversarial '
          'examples: {:.3f}'.format(accuracy))
    display_leg_adv_sample(X_test, X_adv)