import torch
from registry import registry
from eval_settings.eval_setting_base import EvalSetting, StandardDataset, accuracy_topk
from eval_settings.eval_setting_subsample import class_sublist_1_8
imagenet_v2_datasets = [
'imagenetv2-matched-frequency', 'imagenetv2-matched-frequency-format-val',
'imagenetv2-threshold-0.7-format-val', 'imagenetv2-threshold0.7',
'imagenetv2-top-images-format-val', 'imagenetv2-topimages'
]
for imagenet_v2_dataset in imagenet_v2_datasets:
registry.add_eval_setting(
EvalSetting(
name=imagenet_v2_dataset,
dataset=StandardDataset(name=imagenet_v2_dataset),
size=10000,
))
def accuracy_topk_subselected(logits, targets):
targets = torch.tensor(
list(map(lambda x: class_sublist_1_8.index(x), targets)))
return accuracy_topk(logits, targets)
idx_subsample_list = [range(x * 10, (x + 1) * 10) for x in class_sublist_1_8]
idx_subsample_list = sorted(
[item for sublist in idx_subsample_list for item in sublist])
registry.add_eval_setting(
preds = logits.argmax(dim=1)
num_correct, num_total = 0, 0
for pred, image_path in zip(preds, image_paths):
folder = image_path.split('/')[0]
if folder in folder_map:
num_total += 1
if pred in folder_map[folder]:
num_correct += 1
return {'top1': num_correct / num_total * 100}
registry.add_eval_setting(
EvalSetting(
name='objectnet-1.0-beta',
dataset=StandardDataset(name='objectnet-1.0-beta'),
size=18514,
class_sublist=class_sublist,
metrics_fn=objectnet_accuracy,
transform=crop_image,
))
idx_subsample_list = [range(x * 50, (x + 1) * 50) for x in class_sublist]
idx_subsample_list = sorted(
[item for sublist in idx_subsample_list for item in sublist])
def accuracy_topk_subselected(logits, targets):
targets = torch.tensor([class_sublist.index(x) for x in targets])
return accuracy_topk(logits, targets)
'imagenet-c.speckle_noise.2_in-memory': corr_speckle_noise_sev_2,
'imagenet-c.speckle_noise.3_in-memory': corr_speckle_noise_sev_3,
'imagenet-c.speckle_noise.4_in-memory': corr_speckle_noise_sev_4,
'imagenet-c.speckle_noise.5_in-memory': corr_speckle_noise_sev_5,
'imagenet-c.zoom_blur.1_in-memory': corr_zoom_blur_sev_1,
'imagenet-c.zoom_blur.2_in-memory': corr_zoom_blur_sev_2,
'imagenet-c.zoom_blur.3_in-memory': corr_zoom_blur_sev_3,
'imagenet-c.zoom_blur.4_in-memory': corr_zoom_blur_sev_4,
'imagenet-c.zoom_blur.5_in-memory': corr_zoom_blur_sev_5
}
for on_disk_corruption in on_disk_corruptions:
registry.add_eval_setting(
EvalSetting(
name=on_disk_corruption,
dataset=StandardDataset(
name=on_disk_corruption.replace('_on-disk', '')),
size=50000,
))
registry.add_eval_setting(
EvalSetting(
name=on_disk_corruption + '_10percent',
dataset=StandardDataset(
name=on_disk_corruption.replace('_on-disk', '')),
size=5000,
idx_subsample_list=idx_subsample_list_50k_10percent,
parent_eval_setting=on_disk_corruption,
))
def gen_corrupt_batch_gpu(corruption, severity):
eps,
d['step_size'],
d['num_steps'],
d['norm'],
targeted=d['targeted'])
def gen_attack_fn(d):
return lambda images, model: pgd_style_attack(d, images, model)
for pgd_attack_name, d in pgd_attack_params.items():
registry.add_eval_setting(
EvalSetting(
name=pgd_attack_name,
dataset=StandardDataset(name='val'),
size=50000,
perturbation_fn_gpu=gen_attack_fn(d),
adversarial_attack=d,
))
registry.add_eval_setting(
EvalSetting(
name=pgd_attack_name + '_10percent',
dataset=StandardDataset(name='val'),
size=5000,
perturbation_fn_gpu=gen_attack_fn(d),
adversarial_attack=d,
idx_subsample_list=idx_subsample_list_50k_10percent,
parent_eval_setting=pgd_attack_name,
))
from registry import registry
from eval_settings.eval_setting_base import EvalSetting, StandardDataset
registry.add_eval_setting(
EvalSetting(
name='stylized_imagenet',
dataset=StandardDataset(name='stylized_imagenet'),
size=50000,
))