def run_test(self, benchmark, file, expected):
benchmark = benchmark_pool[benchmark]
precomputed_features = Path(__file__).parent / file
with open(precomputed_features, 'rb') as f:
precomputed_features = pickle.load(f)['data']
precomputed_features = precomputed_features.stack(
presentation=['stimulus_path'])
precomputed_paths = set(
map(lstrip_local, precomputed_features['stimulus_path'].values))
# attach stimulus set meta
stimulus_set = benchmark._assembly.stimulus_set
expected_stimulus_paths = list(
map(lstrip_local, [
stimulus_set.get_image(image_id)
for image_id in stimulus_set['image_id']
]))
assert (precomputed_paths == set(expected_stimulus_paths))
for column in stimulus_set.columns:
precomputed_features[column] = 'presentation', stimulus_set[
column].values
precomputed_features = PrecomputedFeatures(
precomputed_features,
visual_degrees=10, # doesn't matter, features are already computed
)
# score
score = benchmark(precomputed_features).raw
assert score.sel(aggregation='center') == expected
def run_test(self, benchmark, file, expected):
benchmark = benchmark_pool[benchmark]
precomputed_features = Path(__file__).parent / file
precomputed_features = BehavioralAssembly(
xr.load_dataarray(precomputed_features))
precomputed_features = precomputed_features.stack(
presentation=['stimulus_path'])
precomputed_paths = list(
map(lambda f: Path(f).name,
precomputed_features['stimulus_path'].values))
# attach stimulus set meta
stimulus_set = benchmark._assembly.stimulus_set
expected_stimulus_paths = [
stimulus_set.get_image(image_id)
for image_id in stimulus_set['image_id']
]
expected_stimulus_paths = list(
map(lambda f: Path(f).name, expected_stimulus_paths))
assert set(precomputed_paths) == set(expected_stimulus_paths)
for column in stimulus_set.columns:
precomputed_features[column] = 'presentation', stimulus_set[
column].values
precomputed_features = PrecomputedFeatures(
precomputed_features,
visual_degrees=10, # doesn't matter, features are already computed
)
# score
score = benchmark(precomputed_features).raw
assert score.sel(aggregation='center') == expected
def test_random_time():
benchmark = DicarloKar2019OST()
rnd = RandomState(0)
stimuli = benchmark._assembly.stimulus_set
source = DataAssembly(rnd.rand(len(stimuli), 5, 5),
coords={
'image_id':
('presentation', stimuli['image_id']),
'image_label':
('presentation', stimuli['image_label']),
'truth': ('presentation', stimuli['truth']),
'neuroid_id': ('neuroid', list(range(5))),
'layer': ('neuroid', ['test'] * 5),
'time_bin_start':
('time_bin', [70, 90, 110, 130, 150]),
'time_bin_end':
('time_bin', [90, 110, 130, 150, 170]),
},
dims=['presentation', 'neuroid', 'time_bin'])
source.name = __name__ + ".test_notime"
source = {benchmark._assembly.stimulus_set.identifier: source}
score = benchmark(PrecomputedFeatures(source, visual_degrees=8))
assert np.isnan(score.sel(aggregation='center')) # not a temporal model
assert np.isnan(
score.raw.sel(aggregation='center')) # not a temporal model
assert score.attrs['ceiling'].sel(aggregation='center') == approx(.79)
def test_self_rdm(self, benchmark, visual_degrees, expected):
benchmark = benchmark_pool[benchmark]
source = benchmark._assembly
score = benchmark(
PrecomputedFeatures(source, visual_degrees=visual_degrees)).raw
assert score.sel(aggregation='center') == expected
raw_values = score.attrs['raw']
assert hasattr(raw_values, 'split')
assert len(raw_values['split']) == 10
def test_Kar2019ost_cornet_s(self):
benchmark = benchmark_pool['dicarlo.Kar2019-ost']
precomputed_features = Path(__file__).parent / 'cornet_s-kar2019.pkl'
with open(precomputed_features, 'rb') as f:
precomputed_features = pickle.load(f)['data']
precomputed_features = PrecomputedFeatures(precomputed_features,
visual_degrees=8)
# score
score = benchmark(precomputed_features).raw
assert score.sel(aggregation='center') == approx(.316, abs=.005)
def test_Kar2019ost_cornet_s(self):
benchmark = benchmark_pool['dicarlo.Kar2019-ost']
precomputed_features = Path(__file__).parent / 'cornet_s-kar2019.nc'
precomputed_features = BehavioralAssembly(
xr.load_dataarray(precomputed_features))
precomputed_features = PrecomputedFeatures(precomputed_features,
visual_degrees=8)
# score
score = benchmark(precomputed_features).raw
assert score.sel(aggregation='center') == approx(.316, abs=.005)
def test_Rajalingham2018public(self):
# load features
precomputed_features = Path(
__file__).parent / 'CORnetZ-rajalingham2018public.pkl'
with open(precomputed_features, 'rb') as f:
precomputed_features = pickle.load(f)['data']
precomputed_features = PrecomputedFeatures(
precomputed_features,
visual_degrees=8, # doesn't matter, features are already computed
)
# score
benchmark = benchmark_pool['dicarlo.Rajalingham2018public-i2n']
score = benchmark(precomputed_features).raw
assert score.sel(aggregation='center') == approx(.136923, abs=.005)
def test_Rajalingham2018public(self):
benchmark = benchmark_pool['dicarlo.Rajalingham2018public-i2n']
# load features
precomputed_features = Path(
__file__).parent / 'CORnetZ-rajalingham2018public.nc'
precomputed_features = BehavioralAssembly(
xr.load_dataarray(precomputed_features))
precomputed_features = PrecomputedFeatures(
precomputed_features,
visual_degrees=8, # doesn't matter, features are already computed
)
# score
score = benchmark(precomputed_features).raw
assert score.sel(aggregation='center') == approx(.136923, abs=.005)
def run_test_properties(self, benchmark, files, expected):
benchmark = benchmark_pool[benchmark]
from brainscore import get_stimulus_set
stimulus_identifiers = np.unique(
np.array([
'dicarlo.Marques2020_blank',
'dicarlo.Marques2020_receptive_field',
'dicarlo.Marques2020_orientation',
benchmark._assembly.stimulus_set.identifier
]))
precomputed_features = {}
for current_stimulus in stimulus_identifiers:
stimulus_set = get_stimulus_set(current_stimulus)
precomputed_features[current_stimulus] = Path(
__file__).parent / files[current_stimulus]
precomputed_features[current_stimulus] = BehavioralAssembly(
xr.load_dataarray(precomputed_features[current_stimulus]))
precomputed_features[current_stimulus] = \
precomputed_features[current_stimulus].stack(presentation=['stimulus_path'])
precomputed_paths = list(
map(
lambda f: Path(f).name,
precomputed_features[current_stimulus]
['stimulus_path'].values))
# attach stimulus set meta
expected_stimulus_paths = [
stimulus_set.get_image(image_id)
for image_id in stimulus_set['image_id']
]
expected_stimulus_paths = list(
map(lambda f: Path(f).name, expected_stimulus_paths))
assert set(precomputed_paths) == set(expected_stimulus_paths)
for column in stimulus_set.columns:
precomputed_features[current_stimulus][
column] = 'presentation', stimulus_set[column].values
precomputed_features = PrecomputedFeatures(precomputed_features,
visual_degrees=8)
# score
score = benchmark(precomputed_features).raw
assert score.sel(aggregation='center') == expected
def test_self(benchmark_ctr, visual_degrees, expected):
benchmark = benchmark_ctr()
source = benchmark._assembly.copy()
score = benchmark(PrecomputedFeatures(source, visual_degrees=visual_degrees)).raw
assert score.sel(aggregation='center') == expected