def __init__(self):
stimulus_set = pd.read_csv(
os.path.join(os.path.dirname(__file__), 'imagenet2012.csv'))
stimulus_set = StimulusSet(stimulus_set)
stimulus_set.image_paths = {
row.image_id: row.filepath
for row in stimulus_set.itertuples()
}
self._stimulus_set = stimulus_set
self._similarity_metric = Accuracy()
ceiling = Score([1, np.nan],
coords={'aggregation': ['center', 'error']},
dims=['aggregation'])
super(Imagenet2012, self).__init__(identifier='fei-fei.Deng2009-top1',
version=1,
ceiling_func=lambda: ceiling,
parent='ImageNet',
bibtex="""@INPROCEEDINGS{5206848,
author={J. {Deng} and W. {Dong} and R. {Socher} and L. {Li} and {Kai Li} and {Li Fei-Fei}},
booktitle={2009 IEEE Conference on Computer Vision and Pattern Recognition},
title={ImageNet: A large-scale hierarchical image database},
year={2009},
volume={},
number={},
pages={248-255},
url = {https://ieeexplore.ieee.org/document/5206848}
}""")
def __init__(self, identifier_suffix, noise_type):
identifier = f'dietterich.Hendrycks2019.{identifier_suffix}'
stimulus_set = brainscore.get_stimulus_set(identifier)
self._stimulus_set = stimulus_set
self._similarity_metric = Accuracy()
self._benchmark_name = identifier
self._noise_type = noise_type
ceiling = Score([1, np.nan], coords={'aggregation': ['center', 'error']}, dims=['aggregation'])
super(Imagenet_C_Individual, self).__init__(identifier=f"{identifier}-top1", version=1,
ceiling_func=lambda: ceiling,
parent=f'dietterich.Hendrycks2019-{noise_type}-top1',
bibtex=BIBTEX)
def __init__(self, stimulus_set, noise_level, noise_type, noise_category):
self.stimulus_set = stimulus_set[stimulus_set['noise_level'] ==
noise_level]
self.noise_level = noise_level
self.noise_type = noise_type
self.benchmark_name = f'dietterich.Hendrycks2019-{noise_category}-{noise_type}-{noise_level}-top1'
self._similarity_metric = Accuracy()
ceiling = Score([1, np.nan],
coords={'aggregation': ['center', 'error']},
dims=['aggregation'])
super(Imagenet_C_Individual, self).__init__(
identifier=self.benchmark_name,
version=2,
ceiling_func=lambda: ceiling,
parent=
f'dietterich.Hendrycks2019-{noise_category}-{noise_type}-top1',
bibtex=BIBTEX)
def __init__(self):
stimulus_set = pd.read_csv(
os.path.join(os.path.dirname(__file__), 'imagenet2012.csv'))
stimulus_set = StimulusSet(stimulus_set)
stimulus_set.image_paths = {
row.image_id: row.filepath
for row in stimulus_set.itertuples()
}
self._stimulus_set = stimulus_set
self._similarity_metric = Accuracy()
ceiling = Score([1, np.nan],
coords={'aggregation': ['center', 'error']},
dims=['aggregation'])
super(Imagenet2012, self).__init__(
identifier='fei-fei.Deng2009-top1',
version=1,
ceiling_func=lambda: ceiling,
parent='ImageNet',
paper_link="https://ieeexplore.ieee.org/abstract/document/5206848")
from collections import OrderedDict
import numpy as np
from brainio_base.assemblies import walk_coords, array_is_element, BehavioralAssembly
from numpy.random.mtrand import RandomState
from tqdm import tqdm
from brainscore.metrics import Score
from brainscore.metrics.accuracy import Accuracy
from brainscore.metrics.transformations import CrossValidationSingle
from brainscore.utils import fullname
from neural_nlp.analyze.data.ceiling import average_subregions
from neural_nlp.neural_data.fmri import load_voxels
from result_caching import store
accuracy = Accuracy()
def fit_decode(train, test):
train, test = train.transpose('presentation', 'neuroid'), test.transpose(
'presentation', 'neuroid')
decoder = TFProbabilitiesClassifier()
decoder.fit(train, train['story'])
train_acc = evaluate(decoder, train)
test_acc = evaluate(decoder, test)
train_acc, test_acc = train_acc.expand_dims(
'train_test'), test_acc.expand_dims('train_test')
train_acc['train_test'], test_acc['train_test'] = ['train'], ['test']
acc = Score.merge(train_acc, test_acc)
return acc