def evaluate_bird_or_bicycle_model(model_fn, dataset_iter=None, model_name=None):
  """
  Evaluates a bird_or_bicycle classifier on a default set of attacks and creates plots
  :param model_fn: A function mapping images to logits
  :param dataset_iter: An iterable that returns (batched_images, batched_labels, image_ids)
  :param model_name: An optional model_fn name
  """
  if dataset_iter is None:
    dataset_iter = bird_or_bicycle.get_iterator('test')

  bird_or_bicycle_shape = (224, 224, 3)
  bird_or_bicycle_spatial_limits = [18, 18, 30]
  bird_or_bicycle_black_border_size = 20  # TODO: What should the border size be here?

  attack_list = [
    attacks.CleanData(),

    attacks.SpatialGridAttack(
      image_shape_hwc=bird_or_bicycle_shape,
      spatial_limits=bird_or_bicycle_spatial_limits,
      grid_granularity=[5, 5, 31],
      black_border_size=bird_or_bicycle_black_border_size,
    ),

    attacks.SpsaWithRandomSpatialAttack(
      model_fn,
      image_shape_hwc=bird_or_bicycle_shape,
      spatial_limits=bird_or_bicycle_spatial_limits,
      black_border_size=0,
      epsilon=(16. / 255),
      num_steps=200,
      batch_size=32
    ),
  ]

  boundary_attack = attacks.BoundaryWithRandomSpatialAttack(
    model_fn,
    max_l2_distortion=10,
    label_to_examples=_get_bird_or_bicycle_label_to_examples(),
    spatial_limits=bird_or_bicycle_spatial_limits,
    black_border_size=bird_or_bicycle_black_border_size,
    image_shape_hwc=bird_or_bicycle_shape,
  )

  # We limit the boundary attack to the first 100 datapoints to speed up eval
  boundary_attack._stop_after_n_datapoints = 100
  attack_list.append(boundary_attack)

  attack_list = [attacks.CleanData()]

  return evaluate_two_class_unambiguous_model(
    model_fn, dataset_iter,
    model_name=model_name,
    attack_list=attack_list)
예제 #2
0
def evaluate_two_class_mnist_model(model_fn,
                                   dataset_iter=None,
                                   model_name=None):
    """
  Evaluates a two-class MNIST model_fn on a default set of attacks and creates plots
  :param model_fn: A function mapping images to logits
  :param dataset_iter: An iterable that returns (batched_images, batched_labels)
  :param model_name: An optional model_fn name
  """

    if dataset_iter is None:
        dataset_iter = mnist_utils.get_two_class_iterator('test',
                                                          num_datapoints=128,
                                                          batch_size=128)

    mnist_spatial_limits = [10, 10, 10]
    mnist_shape = (28, 28, 1)
    mnist_black_border_size = 4

    attack_list = [
        attacks.CleanData(),
        attacks.SpatialGridAttack(
            grid_granularity=[5, 5, 11],
            valid_check=mnist_utils.mnist_valid_check,
            spatial_limits=mnist_spatial_limits,
            black_border_size=mnist_black_border_size,
            image_shape_hwc=mnist_shape,
        ),
        attacks.SpsaWithRandomSpatialAttack(
            model_fn,
            epsilon=0.3,
            valid_check=mnist_utils.mnist_valid_check,
            spatial_limits=mnist_spatial_limits,
            black_border_size=mnist_black_border_size,
            image_shape_hwc=mnist_shape,
            num_steps=2000,
            batch_size=512,
        ),
        attacks.BoundaryWithRandomSpatialAttack(
            model_fn,
            max_l2_distortion=4,
            valid_check=mnist_utils.mnist_valid_check,
            label_to_examples=_get_mnist_labels_to_examples(),
            spatial_limits=mnist_spatial_limits,
            black_border_size=mnist_black_border_size,
            image_shape_hwc=mnist_shape,
        )
    ]

    return evaluate_two_class_unambiguous_model(model_fn,
                                                dataset_iter,
                                                model_name=model_name,
                                                attack_list=attack_list)
def test_two_class_mnist_accuracy():
    """ Train an mnist model on a subset of mnist and then evaluate it
  on small attacks *on the training set*.
  """
    model_fn = train_overfit_classifier(num_batches=32, batch_size=1)
    dataset_iter = mnist_utils.get_two_class_iterator('train',
                                                      num_datapoints=32,
                                                      batch_size=1)

    mnist_spatial_limits = [10, 10, 10]
    mnist_shape = (28, 28, 1)
    mnist_black_border_size = 4

    attack_list = [
        attacks.CleanData(),
        attacks.SpsaAttack(
            model_fn,
            epsilon=0.3,
            image_shape_hwc=mnist_shape,
        ),
        attacks.SpsaWithRandomSpatialAttack(
            model_fn,
            epsilon=0.3,
            spatial_limits=mnist_spatial_limits,
            black_border_size=mnist_black_border_size,
            image_shape_hwc=mnist_shape,
        ),
        attacks.SpatialGridAttack(
            grid_granularity=[5, 5, 11],
            valid_check=mnist_utils.mnist_valid_check,
            spatial_limits=mnist_spatial_limits,
            black_border_size=mnist_black_border_size,
            image_shape_hwc=mnist_shape,
        ),
    ]

    boundary_attack = attacks.BoundaryWithRandomSpatialAttack(
        model_fn,
        max_l2_distortion=4,
        label_to_examples=eval_kit._get_mnist_labels_to_examples(),
        spatial_limits=mnist_spatial_limits,
        black_border_size=mnist_black_border_size,
        image_shape_hwc=mnist_shape,
    )

    # We limit the boundary attack to the first datapoint to speed up eval
    boundary_attack._stop_after_n_datapoints = 1
    attack_list.append(boundary_attack)

    results = eval_kit.evaluate_two_class_unambiguous_model(
        model_fn,
        data_iter=dataset_iter,
        model_name="overfit_mnist",
        attack_list=attack_list)

    # Make sure that clean data has high accuracy
    assert results['clean']['accuracy@100'] >= 0.9

    # Make sure that attacks reduce accuracy
    assert results['spatial_grid']['accuracy@100'] <= 0.7
    assert results['spsa']['accuracy@100'] <= 0.6
    assert results['spsa_with_random_spatial']['accuracy@100'] <= 0.5

    # Run a smoke test on all attacks with a batch size of one
    # TODO: Split this into a separate test
    dataset_iter = mnist_utils.get_two_class_iterator('train',
                                                      num_datapoints=1,
                                                      batch_size=1)
    eval_kit.evaluate_two_class_mnist_model(model_fn,
                                            dataset_iter=dataset_iter,
                                            model_name="overfit_mnist")