def _get_bird_or_bicycle_label_to_examples():
dataset_iter = bird_or_bicycle.get_iterator('test')
label_to_examples = {0: [], 1: []}
for x_np, y_np, _ in dataset_iter:
for x, label in zip(x_np, y_np):
label_to_examples[label].append(x)
return label_to_examples
def test_spatial_speed():
# Keras isn't being found by travis for some reason
from tensorflow.keras.applications.resnet50 import preprocess_input
# Variable scoping lets us use our _graph in our model_fn
_graph = tf.Graph()
with _graph.as_default():
k_model = tf.keras.applications.resnet50.ResNet50(
include_top=True, weights='imagenet', input_tensor=None,
input_shape=None, pooling=None, classes=1000)
def undefended_keras_model_fn(x_np):
"""A normal keras resnet that was pretrained on ImageNet"""
x_np = preprocess_input(x_np * 255)
with _graph.as_default():
prob1000 = k_model.predict_on_batch(x_np) / 10
# Keras returns softmax-ed probs, we convert them back to logits
fake_logits1000 = np.log(prob1000)
bird_max_logit = np.max(
fake_logits1000[:, CLASS_NAME_TO_IMAGENET_CLASS['bird']], axis=1)
bicycle_max_logit = np.max(
fake_logits1000[:, CLASS_NAME_TO_IMAGENET_CLASS['bicycle']], axis=1)
two_class_logits = np.concatenate((
bicycle_max_logit[:, None],
bird_max_logit[:, None]),
axis=1)
return two_class_logits
# Set up standard attack params
bird_or_bicycle_shape = (224, 224, 3)
bird_or_bicycle_spatial_limits = [18, 18, 30]
bird_or_bicycle_black_border_size = 20
grid_granularity = [5, 5, 31]
model_fn = undefended_keras_model_fn
spatial_attack = attacks.FastSpatialGridAttack(
model_fn,
image_shape_hwc=bird_or_bicycle_shape,
spatial_limits=bird_or_bicycle_spatial_limits,
grid_granularity=grid_granularity,
black_border_size=bird_or_bicycle_black_border_size,
)
ds_size = 32
spatial_attack._stop_after_n_datapoints = ds_size
dataset_iter = bird_or_bicycle.get_iterator(
'train', batch_size=32, verify_dataset=False)
return evaluate_two_class_unambiguous_model(
model_fn, dataset_iter,
model_name='undefended_keras_resnet_test_spatial',
attack_list=[spatial_attack])
def evaluate_bird_or_bicycle_model(model_fn, dataset_iter=None, model_name=None):
"""
Evaluates a bird_or_bicycle classifier on a default set of attacks and creates plots
:param model_fn: A function mapping images to logits
:param dataset_iter: An iterable that returns (batched_images, batched_labels, image_ids)
:param model_name: An optional model_fn name
"""
if dataset_iter is None:
dataset_iter = bird_or_bicycle.get_iterator('test')
bird_or_bicycle_shape = (224, 224, 3)
bird_or_bicycle_spatial_limits = [18, 18, 30]
bird_or_bicycle_black_border_size = 20 # TODO: What should the border size be here?
attack_list = [
attacks.CleanData(),
attacks.SpatialGridAttack(
image_shape_hwc=bird_or_bicycle_shape,
spatial_limits=bird_or_bicycle_spatial_limits,
grid_granularity=[5, 5, 31],
black_border_size=bird_or_bicycle_black_border_size,
),
attacks.SpsaWithRandomSpatialAttack(
model_fn,
image_shape_hwc=bird_or_bicycle_shape,
spatial_limits=bird_or_bicycle_spatial_limits,
black_border_size=0,
epsilon=(16. / 255),
num_steps=200,
batch_size=32
),
]
boundary_attack = attacks.BoundaryWithRandomSpatialAttack(
model_fn,
max_l2_distortion=10,
label_to_examples=_get_bird_or_bicycle_label_to_examples(),
spatial_limits=bird_or_bicycle_spatial_limits,
black_border_size=bird_or_bicycle_black_border_size,
image_shape_hwc=bird_or_bicycle_shape,
)
# We limit the boundary attack to the first 100 datapoints to speed up eval
boundary_attack._stop_after_n_datapoints = 100
attack_list.append(boundary_attack)
attack_list = [attacks.CleanData()]
return evaluate_two_class_unambiguous_model(
model_fn, dataset_iter,
model_name=model_name,
attack_list=attack_list)
def test_common_corruptions():
model_fn = create_undefended_keras_model_fn()
severity = 1
spatial_attack = attacks.CommonCorruptionsAttack(severity=severity)
ds_size = 10
spatial_attack._stop_after_n_datapoints = ds_size
dataset_iter = bird_or_bicycle.get_iterator('train',
batch_size=2,
verify_dataset=False)
return evaluate_two_class_unambiguous_model(
model_fn,
dataset_iter,
model_name='test_common_corruption,severity=%s' % severity,
attack_list=[spatial_attack])
def test_simple_spatial():
# Set up standard attack params
bird_or_bicycle_spatial_limits = [18, 18, 30]
grid_granularity = [5, 5, 31]
model_fn = create_undefended_keras_model_fn()
spatial_attack = attacks.SimpleSpatialAttack(
spatial_limits=bird_or_bicycle_spatial_limits,
grid_granularity=grid_granularity,
black_border_frac=0.15,
)
ds_size = 4
spatial_attack._stop_after_n_datapoints = ds_size
dataset_iter = bird_or_bicycle.get_iterator('train',
batch_size=2,
verify_dataset=False)
evaluate_two_class_unambiguous_model(model_fn,
dataset_iter,
model_name='test_spatial',
attack_list=[spatial_attack])
def get_tiny_iterator():
"""An iterator with a single sample"""
dataset_iter = bird_or_bicycle.get_iterator('test', batch_size=1)
return itertools.islice(dataset_iter, 1)