if logits[i, :].argmax().item() == target: correct_per_class[target] += 1 balanced_accuracy = np.mean( [correct_per_class[c] / images_per_class[c] for c in images_per_class]) accuracy = sum(correct_per_class.values()) / sum(images_per_class.values()) return {'top1': accuracy, 'top1_balanced': balanced_accuracy} class_sublist = [int(x) for x in SORTED_ILSVRC_INDICES] registry.add_eval_setting( EvalSetting( name='openimages_test_ilsvrc_subset', dataset=StandardDataset(name='openimages-test'), idx_subsample_list=OPENIMAGES_TEST_ILSVRC_INDICES, size=23104, metrics_fn=validate_openimages, class_sublist=class_sublist, )) idx_subsample_list = [range(x * 50, (x + 1) * 50) for x in class_sublist] idx_subsample_list = sorted( [item for sublist in idx_subsample_list for item in sublist]) def accuracy_topk_subselected(logits, targets): targets = torch.tensor([class_sublist.index(x) for x in targets]) return accuracy_topk(logits, targets)
logits.numpy(), merge_op) metrics_dict = score_predictions(logits_projected, targets.numpy()) metrics_dict['merge_op'] = merge_op return metrics_dict with open((pathlib.Path(__file__).parent / 'imagenet-vid-robust_metadata/class_idx_map.json').resolve()) as f: class_sublist = sorted(list(map(int, json.loads(f.read()).keys()))) registry.add_eval_setting( EvalSetting( name='imagenet-vid-robust', dataset=StandardDataset(name='imagenet-vid-robust'), size=1109, metrics_fn=validate_vid_robust, class_sublist=class_sublist, )) idx_subsample_list = [range(x * 50, (x + 1) * 50) for x in class_sublist] idx_subsample_list = sorted( [item for sublist in idx_subsample_list for item in sublist]) registry.add_eval_setting( EvalSetting( name='val-on-vid-robust-classes', dataset=StandardDataset(name='val'), size=14400, metrics_fn=validate_val_on_vid_robust_classes, class_sublist=class_sublist,
307, 308, 309, 310, 311, 313, 314, 315, 317, 319, 323, 324, 326, 327, 330, 334, 335, 336, 347, 361, 363, 372, 378, 386, 397, 400, 401, 402, 404, 407, 411, 416, 417, 420, 425, 428, 430, 437, 438, 445, 456, 457, 461, 462, 470, 472, 483, 486, 488, 492, 496, 514, 516, 528, 530, 539, 542, 543, 549, 552, 557, 561, 562, 569, 572, 573, 575, 579, 589, 606, 607, 609, 614, 626, 627, 640, 641, 642, 643, 658, 668, 677, 682, 684, 687, 701, 704, 719, 736, 746, 749, 752, 758, 763, 765, 768, 773, 774, 776, 779, 780, 786, 792, 797, 802, 803, 804, 813, 815, 820, 823, 831, 833, 835, 839, 845, 847, 850, 859, 862, 870, 879, 880, 888, 890, 897, 900, 907, 913, 924, 932, 933, 934, 937, 943, 945, 947, 951, 954, 956, 957, 959, 971, 972, 980, 981, 984, 986, 987, 988 ] registry.add_eval_setting( EvalSetting( name='imagenet-a', dataset=StandardDataset(name='imagenet-a'), size=7500, class_sublist=class_sublist, )) idx_subsample_list = [range(x * 50, (x + 1) * 50) for x in class_sublist] idx_subsample_list = sorted( [item for sublist in idx_subsample_list for item in sublist]) def accuracy_topk_subselected(logits, targets): targets = torch.tensor([class_sublist.index(x) for x in targets]) return accuracy_topk(logits, targets) registry.add_eval_setting( EvalSetting(
from registry import registry from eval_settings.eval_setting_base import EvalSetting, StandardDataset from eval_settings.image_utils import greyscale def corrupt_greyscale(image): return greyscale(image) registry.add_eval_setting( EvalSetting( name = 'greyscale', dataset = StandardDataset(name='val'), size = 50000, perturbation_fn_cpu = corrupt_greyscale, ) )
import torch from registry import registry from eval_settings.eval_setting_base import EvalSetting, StandardDataset, accuracy_topk from eval_settings.eval_setting_subsample import class_sublist_1_8 registry.add_eval_setting( EvalSetting( name='val', dataset=StandardDataset(name='val'), size=50000, )) def accuracy_topk_subselected(logits, targets): targets = torch.tensor([class_sublist_1_8.index(x) for x in targets]) return accuracy_topk(logits, targets) idx_subsample_list = [range(x * 50, (x + 1) * 50) for x in class_sublist_1_8] idx_subsample_list = sorted( [item for sublist in idx_subsample_list for item in sublist]) registry.add_eval_setting( EvalSetting( name='val_subsampled_class_1_8', dataset=StandardDataset(name='val'), size=6250, class_sublist=class_sublist_1_8, metrics_fn=accuracy_topk_subselected, idx_subsample_list=idx_subsample_list,
from registry import registry from eval_settings.eval_setting_base import EvalSetting, StandardDataset, accuracy_topk from eval_settings.eval_setting_subsample import class_sublist_1_8 imagenet_v2_datasets = [ 'imagenetv2-matched-frequency', 'imagenetv2-matched-frequency-format-val', 'imagenetv2-threshold-0.7-format-val', 'imagenetv2-threshold0.7', 'imagenetv2-top-images-format-val', 'imagenetv2-topimages' ] for imagenet_v2_dataset in imagenet_v2_datasets: registry.add_eval_setting( EvalSetting( name=imagenet_v2_dataset, dataset=StandardDataset(name=imagenet_v2_dataset), size=10000, )) def accuracy_topk_subselected(logits, targets): targets = torch.tensor( list(map(lambda x: class_sublist_1_8.index(x), targets))) return accuracy_topk(logits, targets) idx_subsample_list = [range(x * 10, (x + 1) * 10) for x in class_sublist_1_8] idx_subsample_list = sorted( [item for sublist in idx_subsample_list for item in sublist]) registry.add_eval_setting(
preds = logits.argmax(dim=1) num_correct, num_total = 0, 0 for pred, image_path in zip(preds, image_paths): folder = image_path.split('/')[0] if folder in folder_map: num_total += 1 if pred in folder_map[folder]: num_correct += 1 return {'top1': num_correct / num_total * 100} registry.add_eval_setting( EvalSetting( name='objectnet-1.0-beta', dataset=StandardDataset(name='objectnet-1.0-beta'), size=18514, class_sublist=class_sublist, metrics_fn=objectnet_accuracy, transform=crop_image, )) idx_subsample_list = [range(x * 50, (x + 1) * 50) for x in class_sublist] idx_subsample_list = sorted( [item for sublist in idx_subsample_list for item in sublist]) def accuracy_topk_subselected(logits, targets): targets = torch.tensor([class_sublist.index(x) for x in targets]) return accuracy_topk(logits, targets) registry.add_eval_setting(
eps, d['step_size'], d['num_steps'], d['norm'], targeted=d['targeted']) def gen_attack_fn(d): return lambda images, model: pgd_style_attack(d, images, model) for pgd_attack_name, d in pgd_attack_params.items(): registry.add_eval_setting( EvalSetting( name=pgd_attack_name, dataset=StandardDataset(name='val'), size=50000, perturbation_fn_gpu=gen_attack_fn(d), adversarial_attack=d, )) registry.add_eval_setting( EvalSetting( name=pgd_attack_name + '_10percent', dataset=StandardDataset(name='val'), size=5000, perturbation_fn_gpu=gen_attack_fn(d), adversarial_attack=d, idx_subsample_list=idx_subsample_list_50k_10percent, parent_eval_setting=pgd_attack_name, ))
'imagenet-c.speckle_noise.2_in-memory': corr_speckle_noise_sev_2, 'imagenet-c.speckle_noise.3_in-memory': corr_speckle_noise_sev_3, 'imagenet-c.speckle_noise.4_in-memory': corr_speckle_noise_sev_4, 'imagenet-c.speckle_noise.5_in-memory': corr_speckle_noise_sev_5, 'imagenet-c.zoom_blur.1_in-memory': corr_zoom_blur_sev_1, 'imagenet-c.zoom_blur.2_in-memory': corr_zoom_blur_sev_2, 'imagenet-c.zoom_blur.3_in-memory': corr_zoom_blur_sev_3, 'imagenet-c.zoom_blur.4_in-memory': corr_zoom_blur_sev_4, 'imagenet-c.zoom_blur.5_in-memory': corr_zoom_blur_sev_5 } for on_disk_corruption in on_disk_corruptions: registry.add_eval_setting( EvalSetting( name=on_disk_corruption, dataset=StandardDataset( name=on_disk_corruption.replace('_on-disk', '')), size=50000, )) registry.add_eval_setting( EvalSetting( name=on_disk_corruption + '_10percent', dataset=StandardDataset( name=on_disk_corruption.replace('_on-disk', '')), size=5000, idx_subsample_list=idx_subsample_list_50k_10percent, parent_eval_setting=on_disk_corruption, )) def gen_corrupt_batch_gpu(corruption, severity): def corrupt_batch_gpu(images, model):
from registry import registry from eval_settings.eval_setting_base import EvalSetting, StandardDataset registry.add_eval_setting( EvalSetting( name='stylized_imagenet', dataset=StandardDataset(name='stylized_imagenet'), size=50000, ))