def _get_bird_or_bicycle_label_to_examples():
dataset_iter = bird_or_bicyle.get_iterator('test')
label_to_examples = {0: [], 1: []}
for x_np, y_np in dataset_iter:
for x, label in zip(x_np, y_np):
label_to_examples[label].append(x)
return label_to_examples
def evaluate_bird_or_bicycle_model(model_fn, dataset_iter=None, model_name=None):
"""
Evaluates a bird_or_bicycle classifier on a default set of attacks and creates plots
:param model_fn: A function mapping images to logits
:param dataset_iter: An iterable that returns (batched_images, batched_labels)
:param model_name: An optional model_fn name
"""
if dataset_iter is None:
dataset_iter = bird_or_bicyle.get_iterator('test')
bird_or_bicycle_shape = (224, 224, 3)
bird_or_bicycle_spatial_limits = [18, 18, 30]
bird_or_bicycle_black_border_size = 20 # TODO: What should the border size be here?
attack_list = [
attacks.CleanData(),
attacks.SpatialGridAttack(
image_shape_hwc=bird_or_bicycle_shape,
spatial_limits=bird_or_bicycle_spatial_limits,
grid_granularity=[5, 5, 31],
black_border_size=bird_or_bicycle_black_border_size,
),
attacks.SpsaWithRandomSpatialAttack(
model_fn,
image_shape_hwc=bird_or_bicycle_shape,
spatial_limits=bird_or_bicycle_spatial_limits,
black_border_size=0,
epsilon=(16. / 255),
num_steps=200,
batch_size=32
),
]
boundary_attack = attacks.BoundaryWithRandomSpatialAttack(
model_fn,
max_l2_distortion=10,
label_to_examples=_get_bird_or_bicycle_label_to_examples(),
spatial_limits=bird_or_bicycle_spatial_limits,
black_border_size=bird_or_bicycle_black_border_size,
image_shape_hwc=bird_or_bicycle_shape,
)
# We limit the boundary attack to the first 100 datapoints to speed up eval
boundary_attack._stop_after_n_datapoints = 100
attack_list.append(boundary_attack)
return evaluate_two_class_unambiguous_model(
model_fn, dataset_iter,
model_name=model_name,
attack_list=attack_list)
def get_tiny_iterator():
"""An iterator with a single sample"""
dataset_iter = bird_or_bicyle.get_iterator('test', batch_size=1)
return itertools.islice(dataset_iter, 1)