images_per_class[target] += 1
if logits[i, :].argmax().item() == target:
correct_per_class[target] += 1
balanced_accuracy = np.mean(
[correct_per_class[c] / images_per_class[c] for c in images_per_class])
accuracy = sum(correct_per_class.values()) / sum(images_per_class.values())
return {'top1': accuracy, 'top1_balanced': balanced_accuracy}
class_sublist = [int(x) for x in SORTED_ILSVRC_INDICES]
registry.add_eval_setting(
EvalSetting(
name='openimages_test_ilsvrc_subset',
dataset=StandardDataset(name='openimages-test'),
idx_subsample_list=OPENIMAGES_TEST_ILSVRC_INDICES,
size=23104,
metrics_fn=validate_openimages,
class_sublist=class_sublist,
))
idx_subsample_list = [range(x * 50, (x + 1) * 50) for x in class_sublist]
idx_subsample_list = sorted(
[item for sublist in idx_subsample_list for item in sublist])
def accuracy_topk_subselected(logits, targets):
targets = torch.tensor([class_sublist.index(x) for x in targets])
return accuracy_topk(logits, targets)
logits_projected = project_imagenet_predictions_imagenet_vid(
logits.numpy(), merge_op)
metrics_dict = score_predictions(logits_projected, targets.numpy())
metrics_dict['merge_op'] = merge_op
return metrics_dict
with open((pathlib.Path(__file__).parent /
'imagenet-vid-robust_metadata/class_idx_map.json').resolve()) as f:
class_sublist = sorted(list(map(int, json.loads(f.read()).keys())))
registry.add_eval_setting(
EvalSetting(
name='imagenet-vid-robust',
dataset=StandardDataset(name='imagenet-vid-robust'),
size=1109,
metrics_fn=validate_vid_robust,
class_sublist=class_sublist,
))
idx_subsample_list = [range(x * 50, (x + 1) * 50) for x in class_sublist]
idx_subsample_list = sorted(
[item for sublist in idx_subsample_list for item in sublist])
registry.add_eval_setting(
EvalSetting(
name='val-on-vid-robust-classes',
dataset=StandardDataset(name='val'),
size=14400,
metrics_fn=validate_val_on_vid_robust_classes,
307, 308, 309, 310, 311, 313, 314, 315, 317, 319, 323, 324, 326, 327, 330,
334, 335, 336, 347, 361, 363, 372, 378, 386, 397, 400, 401, 402, 404, 407,
411, 416, 417, 420, 425, 428, 430, 437, 438, 445, 456, 457, 461, 462, 470,
472, 483, 486, 488, 492, 496, 514, 516, 528, 530, 539, 542, 543, 549, 552,
557, 561, 562, 569, 572, 573, 575, 579, 589, 606, 607, 609, 614, 626, 627,
640, 641, 642, 643, 658, 668, 677, 682, 684, 687, 701, 704, 719, 736, 746,
749, 752, 758, 763, 765, 768, 773, 774, 776, 779, 780, 786, 792, 797, 802,
803, 804, 813, 815, 820, 823, 831, 833, 835, 839, 845, 847, 850, 859, 862,
870, 879, 880, 888, 890, 897, 900, 907, 913, 924, 932, 933, 934, 937, 943,
945, 947, 951, 954, 956, 957, 959, 971, 972, 980, 981, 984, 986, 987, 988
]
registry.add_eval_setting(
EvalSetting(
name='imagenet-a',
dataset=StandardDataset(name='imagenet-a'),
size=7500,
class_sublist=class_sublist,
))
idx_subsample_list = [range(x * 50, (x + 1) * 50) for x in class_sublist]
idx_subsample_list = sorted(
[item for sublist in idx_subsample_list for item in sublist])
def accuracy_topk_subselected(logits, targets):
targets = torch.tensor([class_sublist.index(x) for x in targets])
return accuracy_topk(logits, targets)
registry.add_eval_setting(
import torch
from registry import registry
from eval_settings.eval_setting_base import EvalSetting, StandardDataset, accuracy_topk
from eval_settings.eval_setting_subsample import class_sublist_1_8
registry.add_eval_setting(
EvalSetting(
name='val',
dataset=StandardDataset(name='val'),
size=50000,
))
def accuracy_topk_subselected(logits, targets):
targets = torch.tensor([class_sublist_1_8.index(x) for x in targets])
return accuracy_topk(logits, targets)
idx_subsample_list = [range(x * 50, (x + 1) * 50) for x in class_sublist_1_8]
idx_subsample_list = sorted(
[item for sublist in idx_subsample_list for item in sublist])
registry.add_eval_setting(
EvalSetting(
name='val_subsampled_class_1_8',
dataset=StandardDataset(name='val'),
size=6250,
class_sublist=class_sublist_1_8,
metrics_fn=accuracy_topk_subselected,
idx_subsample_list=idx_subsample_list,
from registry import registry
from eval_settings.eval_setting_base import EvalSetting, StandardDataset, accuracy_topk
from eval_settings.eval_setting_subsample import class_sublist_1_8
imagenet_v2_datasets = [
'imagenetv2-matched-frequency', 'imagenetv2-matched-frequency-format-val',
'imagenetv2-threshold-0.7-format-val', 'imagenetv2-threshold0.7',
'imagenetv2-top-images-format-val', 'imagenetv2-topimages'
]
for imagenet_v2_dataset in imagenet_v2_datasets:
registry.add_eval_setting(
EvalSetting(
name=imagenet_v2_dataset,
dataset=StandardDataset(name=imagenet_v2_dataset),
size=10000,
))
def accuracy_topk_subselected(logits, targets):
targets = torch.tensor(
list(map(lambda x: class_sublist_1_8.index(x), targets)))
return accuracy_topk(logits, targets)
idx_subsample_list = [range(x * 10, (x + 1) * 10) for x in class_sublist_1_8]
idx_subsample_list = sorted(
[item for sublist in idx_subsample_list for item in sublist])
registry.add_eval_setting(
preds = logits.argmax(dim=1)
num_correct, num_total = 0, 0
for pred, image_path in zip(preds, image_paths):
folder = image_path.split('/')[0]
if folder in folder_map:
num_total += 1
if pred in folder_map[folder]:
num_correct += 1
return {'top1': num_correct / num_total * 100}
registry.add_eval_setting(
EvalSetting(
name='objectnet-1.0-beta',
dataset=StandardDataset(name='objectnet-1.0-beta'),
size=18514,
class_sublist=class_sublist,
metrics_fn=objectnet_accuracy,
transform=crop_image,
))
idx_subsample_list = [range(x * 50, (x + 1) * 50) for x in class_sublist]
idx_subsample_list = sorted(
[item for sublist in idx_subsample_list for item in sublist])
def accuracy_topk_subselected(logits, targets):
targets = torch.tensor([class_sublist.index(x) for x in targets])
return accuracy_topk(logits, targets)
'imagenet-c.speckle_noise.2_in-memory': corr_speckle_noise_sev_2,
'imagenet-c.speckle_noise.3_in-memory': corr_speckle_noise_sev_3,
'imagenet-c.speckle_noise.4_in-memory': corr_speckle_noise_sev_4,
'imagenet-c.speckle_noise.5_in-memory': corr_speckle_noise_sev_5,
'imagenet-c.zoom_blur.1_in-memory': corr_zoom_blur_sev_1,
'imagenet-c.zoom_blur.2_in-memory': corr_zoom_blur_sev_2,
'imagenet-c.zoom_blur.3_in-memory': corr_zoom_blur_sev_3,
'imagenet-c.zoom_blur.4_in-memory': corr_zoom_blur_sev_4,
'imagenet-c.zoom_blur.5_in-memory': corr_zoom_blur_sev_5
}
for on_disk_corruption in on_disk_corruptions:
registry.add_eval_setting(
EvalSetting(
name=on_disk_corruption,
dataset=StandardDataset(
name=on_disk_corruption.replace('_on-disk', '')),
size=50000,
))
registry.add_eval_setting(
EvalSetting(
name=on_disk_corruption + '_10percent',
dataset=StandardDataset(
name=on_disk_corruption.replace('_on-disk', '')),
size=5000,
idx_subsample_list=idx_subsample_list_50k_10percent,
parent_eval_setting=on_disk_corruption,
))
def gen_corrupt_batch_gpu(corruption, severity):
def corrupt_batch_gpu(images, model):
from registry import registry
from eval_settings.eval_setting_base import EvalSetting, StandardDataset
registry.add_eval_setting(
EvalSetting(
name='stylized_imagenet',
dataset=StandardDataset(name='stylized_imagenet'),
size=50000,
))