def run_test(self, benchmark, file, expected): benchmark = benchmark_pool[benchmark] precomputed_features = Path(__file__).parent / file with open(precomputed_features, 'rb') as f: precomputed_features = pickle.load(f)['data'] precomputed_features = precomputed_features.stack( presentation=['stimulus_path']) precomputed_paths = set( map(lstrip_local, precomputed_features['stimulus_path'].values)) # attach stimulus set meta stimulus_set = benchmark._assembly.stimulus_set expected_stimulus_paths = list( map(lstrip_local, [ stimulus_set.get_image(image_id) for image_id in stimulus_set['image_id'] ])) assert (precomputed_paths == set(expected_stimulus_paths)) for column in stimulus_set.columns: precomputed_features[column] = 'presentation', stimulus_set[ column].values precomputed_features = PrecomputedFeatures( precomputed_features, visual_degrees=10, # doesn't matter, features are already computed ) # score score = benchmark(precomputed_features).raw assert score.sel(aggregation='center') == expected
def run_test(self, benchmark, file, expected): benchmark = benchmark_pool[benchmark] precomputed_features = Path(__file__).parent / file precomputed_features = BehavioralAssembly( xr.load_dataarray(precomputed_features)) precomputed_features = precomputed_features.stack( presentation=['stimulus_path']) precomputed_paths = list( map(lambda f: Path(f).name, precomputed_features['stimulus_path'].values)) # attach stimulus set meta stimulus_set = benchmark._assembly.stimulus_set expected_stimulus_paths = [ stimulus_set.get_image(image_id) for image_id in stimulus_set['image_id'] ] expected_stimulus_paths = list( map(lambda f: Path(f).name, expected_stimulus_paths)) assert set(precomputed_paths) == set(expected_stimulus_paths) for column in stimulus_set.columns: precomputed_features[column] = 'presentation', stimulus_set[ column].values precomputed_features = PrecomputedFeatures( precomputed_features, visual_degrees=10, # doesn't matter, features are already computed ) # score score = benchmark(precomputed_features).raw assert score.sel(aggregation='center') == expected
def test_random_time(): benchmark = DicarloKar2019OST() rnd = RandomState(0) stimuli = benchmark._assembly.stimulus_set source = DataAssembly(rnd.rand(len(stimuli), 5, 5), coords={ 'image_id': ('presentation', stimuli['image_id']), 'image_label': ('presentation', stimuli['image_label']), 'truth': ('presentation', stimuli['truth']), 'neuroid_id': ('neuroid', list(range(5))), 'layer': ('neuroid', ['test'] * 5), 'time_bin_start': ('time_bin', [70, 90, 110, 130, 150]), 'time_bin_end': ('time_bin', [90, 110, 130, 150, 170]), }, dims=['presentation', 'neuroid', 'time_bin']) source.name = __name__ + ".test_notime" source = {benchmark._assembly.stimulus_set.identifier: source} score = benchmark(PrecomputedFeatures(source, visual_degrees=8)) assert np.isnan(score.sel(aggregation='center')) # not a temporal model assert np.isnan( score.raw.sel(aggregation='center')) # not a temporal model assert score.attrs['ceiling'].sel(aggregation='center') == approx(.79)
def test_self_rdm(self, benchmark, visual_degrees, expected): benchmark = benchmark_pool[benchmark] source = benchmark._assembly score = benchmark( PrecomputedFeatures(source, visual_degrees=visual_degrees)).raw assert score.sel(aggregation='center') == expected raw_values = score.attrs['raw'] assert hasattr(raw_values, 'split') assert len(raw_values['split']) == 10
def test_Kar2019ost_cornet_s(self): benchmark = benchmark_pool['dicarlo.Kar2019-ost'] precomputed_features = Path(__file__).parent / 'cornet_s-kar2019.pkl' with open(precomputed_features, 'rb') as f: precomputed_features = pickle.load(f)['data'] precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=8) # score score = benchmark(precomputed_features).raw assert score.sel(aggregation='center') == approx(.316, abs=.005)
def test_Kar2019ost_cornet_s(self): benchmark = benchmark_pool['dicarlo.Kar2019-ost'] precomputed_features = Path(__file__).parent / 'cornet_s-kar2019.nc' precomputed_features = BehavioralAssembly( xr.load_dataarray(precomputed_features)) precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=8) # score score = benchmark(precomputed_features).raw assert score.sel(aggregation='center') == approx(.316, abs=.005)
def test_Rajalingham2018public(self): # load features precomputed_features = Path( __file__).parent / 'CORnetZ-rajalingham2018public.pkl' with open(precomputed_features, 'rb') as f: precomputed_features = pickle.load(f)['data'] precomputed_features = PrecomputedFeatures( precomputed_features, visual_degrees=8, # doesn't matter, features are already computed ) # score benchmark = benchmark_pool['dicarlo.Rajalingham2018public-i2n'] score = benchmark(precomputed_features).raw assert score.sel(aggregation='center') == approx(.136923, abs=.005)
def test_Rajalingham2018public(self): benchmark = benchmark_pool['dicarlo.Rajalingham2018public-i2n'] # load features precomputed_features = Path( __file__).parent / 'CORnetZ-rajalingham2018public.nc' precomputed_features = BehavioralAssembly( xr.load_dataarray(precomputed_features)) precomputed_features = PrecomputedFeatures( precomputed_features, visual_degrees=8, # doesn't matter, features are already computed ) # score score = benchmark(precomputed_features).raw assert score.sel(aggregation='center') == approx(.136923, abs=.005)
def run_test_properties(self, benchmark, files, expected): benchmark = benchmark_pool[benchmark] from brainscore import get_stimulus_set stimulus_identifiers = np.unique( np.array([ 'dicarlo.Marques2020_blank', 'dicarlo.Marques2020_receptive_field', 'dicarlo.Marques2020_orientation', benchmark._assembly.stimulus_set.identifier ])) precomputed_features = {} for current_stimulus in stimulus_identifiers: stimulus_set = get_stimulus_set(current_stimulus) precomputed_features[current_stimulus] = Path( __file__).parent / files[current_stimulus] precomputed_features[current_stimulus] = BehavioralAssembly( xr.load_dataarray(precomputed_features[current_stimulus])) precomputed_features[current_stimulus] = \ precomputed_features[current_stimulus].stack(presentation=['stimulus_path']) precomputed_paths = list( map( lambda f: Path(f).name, precomputed_features[current_stimulus] ['stimulus_path'].values)) # attach stimulus set meta expected_stimulus_paths = [ stimulus_set.get_image(image_id) for image_id in stimulus_set['image_id'] ] expected_stimulus_paths = list( map(lambda f: Path(f).name, expected_stimulus_paths)) assert set(precomputed_paths) == set(expected_stimulus_paths) for column in stimulus_set.columns: precomputed_features[current_stimulus][ column] = 'presentation', stimulus_set[column].values precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=8) # score score = benchmark(precomputed_features).raw assert score.sel(aggregation='center') == expected
def test_self(benchmark_ctr, visual_degrees, expected): benchmark = benchmark_ctr() source = benchmark._assembly.copy() score = benchmark(PrecomputedFeatures(source, visual_degrees=visual_degrees)).raw assert score.sel(aggregation='center') == expected