def collect_data(): n_neurons = 87 # Spatial frequency selective data spatial_frequency_selective_bins = np.linspace(0, 1, num=3) spatial_frequency_selective_hist = np.array([14, 73]) spatial_frequency_selective = np.concatenate((np.zeros(spatial_frequency_selective_hist[0]), np.ones(spatial_frequency_selective_hist[1]))).reshape((-1,1)) # Spatial frequency bandwidth data spatial_frequency_bandwidth_bins = np.linspace(0, 100, num=11) spatial_frequency_bandwidth_simple_hist = np.array([0, 0, 0, 6, 5, 10, 7, 2, 0, 0]) spatial_frequency_bandwidth_complex_hist = np.array([0, 3, 4, 10, 17, 6, 2, 1, 0, 0]) spatial_frequency_bandwidth_hist = (spatial_frequency_bandwidth_simple_hist + spatial_frequency_bandwidth_complex_hist) spatial_frequency_bandwidth = gen_sample(spatial_frequency_bandwidth_hist, spatial_frequency_bandwidth_bins, scale='linear') filler = np.zeros((n_neurons - spatial_frequency_bandwidth_hist.sum(), 1)) filler[:] = np.nan spatial_frequency_bandwidth = np.concatenate((filler, spatial_frequency_bandwidth)) # Create DataAssembly with single neuronal properties and bin information assembly = np.concatenate((spatial_frequency_selective, spatial_frequency_bandwidth), axis=1) assembly = DataAssembly(assembly, coords={'neuroid_id': ('neuroid', range(assembly.shape[0])), 'region': ('neuroid', ['V1'] * assembly.shape[0]), 'neuronal_property': ['spatial_frequency_selective', 'spatial_frequency_bandwidth']}, dims=['neuroid', 'neuronal_property']) assembly.attrs['number_of_trials'] = 20 for p in assembly.coords['neuronal_property'].values: assembly.attrs[p+'_bins'] = eval(p+'_bins') return assembly
def test_random_time(): benchmark = DicarloKar2019OST() rnd = RandomState(0) stimuli = benchmark._assembly.stimulus_set source = DataAssembly(rnd.rand(len(stimuli), 5, 5), coords={ 'image_id': ('presentation', stimuli['image_id']), 'image_label': ('presentation', stimuli['image_label']), 'truth': ('presentation', stimuli['truth']), 'neuroid_id': ('neuroid', list(range(5))), 'layer': ('neuroid', ['test'] * 5), 'time_bin_start': ('time_bin', [70, 90, 110, 130, 150]), 'time_bin_end': ('time_bin', [90, 110, 130, 150, 170]), }, dims=['presentation', 'neuroid', 'time_bin']) source.name = __name__ + ".test_notime" source = {benchmark._assembly.stimulus_set.identifier: source} score = benchmark(PrecomputedFeatures(source, visual_degrees=8)) assert np.isnan(score.sel(aggregation='center')) # not a temporal model assert np.isnan( score.raw.sel(aggregation='center')) # not a temporal model assert score.attrs['ceiling'].sel(aggregation='center') == approx(.79)
def test_single_element(self): d = DataAssembly([0], coords={ 'coordA': ('dim', [0]), 'coordB': ('dim', [1]) }, dims=['dim']) d.sel(coordA=0) d.sel(coordB=1)
def test_unique_values_swappeddims(self): d = DataAssembly([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], coords={ 'a': ['a', 'b', 'c', 'd'], 'b': ['x', 'y', 'z'] }, dims=['a', 'b']) g = d.multi_dim_apply(['b', 'a'], lambda x, **_: x) assert g.equals(d)
def test_single_dimension_int(self): d = DataAssembly([[1, 2, 3], [4, 5, 6]], coords={ 'a': [1, 2], 'b': [3, 4, 5] }, dims=['a', 'b']) g = d.multi_groupby(['a']).mean() assert g.equals( DataAssembly([2., 5.], coords={'a': [1, 2]}, dims=['a']))
def test_single_dimension(self): d = DataAssembly([[1, 2, 3], [4, 5, 6]], coords={ 'a': ['a', 'b'], 'b': ['x', 'y', 'z'] }, dims=['a', 'b']) g = d.multi_groupby(['a']).mean() assert g.equals( DataAssembly([2, 5], coords={'a': ['a', 'b']}, dims=['a']))
def load_voxels(bold_shift_seconds=4): assembly = load_voxel_data(bold_shift_seconds=bold_shift_seconds) assembly = DataAssembly(assembly) stimulus_set = NaturalisticStories()() stimulus_set, assembly = _align_stimuli_recordings(stimulus_set, assembly) assert set(assembly['stimulus_sentence'].values).issubset( set(stimulus_set['sentence'])) assembly.attrs['stimulus_set'] = stimulus_set assembly.attrs['stimulus_set_name'] = stimulus_set.name return assembly
def extrapolate_neuroid(self, ceilings): # figure out how many extrapolation x points we have. E.g. for Pereira, not all combinations are possible subject_subsamples = list(sorted(set(ceilings['num_subjects'].values))) rng = RandomState(0) bootstrap_params = [] for bootstrap in range(self.num_bootstraps): bootstrapped_scores = [] for num_subjects in subject_subsamples: num_scores = ceilings.sel(num_subjects=num_subjects) # the sub_subjects dimension creates nans, get rid of those num_scores = num_scores.dropna(f'sub_{self.subject_column}') assert set(num_scores.dims) == {f'sub_{self.subject_column}', 'split'} or \ set(num_scores.dims) == {f'sub_{self.subject_column}'} # choose from subject subsets and the splits therein, with replacement for variance choices = num_scores.values.flatten() bootstrapped_score = rng.choice(choices, size=len(choices), replace=True) bootstrapped_scores.append(np.mean(bootstrapped_score)) try: params = self.fit(subject_subsamples, bootstrapped_scores) except RuntimeError: # optimal parameters not found params = [np.nan, np.nan] params = DataAssembly([params], coords={ 'bootstrap': [bootstrap], 'param': ['v0', 'tau0'] }, dims=['bootstrap', 'param']) bootstrap_params.append(params) bootstrap_params = merge_data_arrays(bootstrap_params) # find endpoint and error asymptote_threshold = .0005 interpolation_xs = np.arange(1000) ys = np.array([ v(interpolation_xs, *params) for params in bootstrap_params.values if not np.isnan(params).any() ]) median_ys = np.median(ys, axis=0) diffs = np.diff(median_ys) end_x = np.where(diffs < asymptote_threshold)[0].min( ) # first x where increase smaller than threshold # put together center = np.median(np.array(bootstrap_params)[:, 0]) error = ci_error(ys[:, end_x], center=center) score = Score( [center] + list(error), coords={'aggregation': ['center', 'error_low', 'error_high']}, dims=['aggregation']) score.attrs['raw'] = ceilings score.attrs['bootstrapped_params'] = bootstrap_params score.attrs['endpoint_x'] = DataAssembly(end_x) return score
def test_multi_level(self): d = DataAssembly( [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12], [13, 14, 15], [16, 17, 18]], coords={ 'up': ("a", ['alpha', 'alpha', 'beta', 'beta', 'beta', 'beta']), 'down': ("a", [1, 1, 1, 1, 2, 2]), 'sideways': ('b', ['x', 'y', 'z']) }, dims=['a', 'b']) g = d.multi_dim_apply(['a', 'b'], lambda x, **_: x) assert g.equals(d)
def test_nonindex_coord(self): d = DataAssembly( [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], coords={ 'a': ['a', 'b', 'c', 'd'], 'b': ['x', 'y', 'z'], # additional coordinate that has no index values. # This could e.g. be the result of `.sel(c='remnant')` 'c': 'remnant' }, dims=['a', 'b']) g = d.multi_dim_apply(['a', 'b'], lambda x, **_: x) assert g.equals(d) # also tests that `c` persists
def test_reset_index_levels(): assy = DataAssembly( data=[[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12], [13, 14, 15], [16, 17, 18]], coords={ 'up': ("a", ['alpha', 'alpha', 'beta', 'beta', 'beta', 'beta']), 'down': ("a", [1, 1, 1, 1, 2, 2]), 'sideways': ('b', ['x', 'y', 'z']) }, dims=['a', 'b']) assert assy["a"].variable.level_names == ["up", "down"] assy = assy.reset_index(["up", "down"]) assert get_levels(assy) == []
def test_single_coord(self): d = DataAssembly( [[1, 2, 3], [4, 5, 6]], coords={ 'a': ('multi_dim', ['a', 'b']), 'b': ('multi_dim', ['c', 'c']), 'c': ['x', 'y', 'z'] }, dims=['multi_dim', 'c']) g = d.multi_groupby(['a']).mean() assert g.equals( DataAssembly([2, 5], coords={'multi_dim': ['a', 'b']}, dims=['multi_dim']))
def test_multi_dim(self): d = DataAssembly([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], coords={ 'a': ['a', 'a', 'b', 'b'], 'b': ['x', 'y', 'z'] }, dims=['a', 'b']) g = d.multi_groupby(['a', 'b']).mean() assert g.equals( DataAssembly([2.5, 3.5, 4.5], [8.5, 9.5, 10.5], coords={ 'a': ['a', 'b'], 'b': ['x', 'y', 'z'] }, dims=['a', 'b']))
def test_mean_no_apply_raw(self): score = Score([1, 2], coords={'a': [1, 2]}, dims=['a']) score.attrs['raw'] = DataAssembly([0, 2, 1, 3], coords={'a': [1, 1, 2, 2]}, dims=['a']) mean_score = score.mean('a', _apply_raw=True) assert mean_score.raw == 1.5
def test_sel(self): score = Score([1, 2], coords={'a': [1, 2]}, dims=['a']) score.attrs['raw'] = DataAssembly([0, 2, 1, 3], coords={'a': [1, 1, 2, 2]}, dims=['a']) sel_score = score.sel(a=1) np.testing.assert_array_equal(sel_score.raw['a'], [1, 1])
def load_rdm_sentences(story='Boar', roi_filter='from90to100', bold_shift_seconds=4): timepoint_rdms = load_rdm_timepoints(story, roi_filter) meta_data = load_sentences_meta(story) del meta_data['fullSentence'] meta_data.dropna(inplace=True) mapping_column = 'shiftBOLD_{}sec'.format(bold_shift_seconds) timepoints = meta_data[mapping_column].values.astype(int) # filter and annotate assert all(timepoint in timepoint_rdms['timepoint_left'].values for timepoint in timepoints) timepoint_rdms = timepoint_rdms.sel(timepoint_left=timepoints, timepoint_right=timepoints) # re-interpret timepoints as stimuli coords = {} for coord_name, coord_value in timepoint_rdms.coords.items(): dims = timepoint_rdms.coords[coord_name].dims dims = [ dim if not dim.startswith('timepoint') else 'presentation' for dim in dims ] coords[coord_name] = dims, coord_value.values coords = { **coords, **{ 'stimulus_sentence': ('presentation', meta_data['reducedSentence'].values) } } dims = [ dim if not dim.startswith('timepoint') else 'presentation' for dim in timepoint_rdms.dims ] data = DataAssembly(timepoint_rdms, coords=coords, dims=dims) return data
def __call__(self, assembly): assert len(assembly.dims) == 2 correlations = np.corrcoef(assembly) if assembly.dims[-1] == self._neuroid_dim else np.corrcoef(assembly.T).T coords = {coord: coord_value for coord, coord_value in assembly.coords.items() if coord != self._neuroid_dim} dims = [dim if dim != self._neuroid_dim else assembly.dims[(i - 1) % len(assembly.dims)] for i, dim in enumerate(assembly.dims)] return DataAssembly(correlations, coords=coords, dims=dims)
def test_mean(self): score = Score([1, 2], coords={'a': [1, 2]}, dims=['a']) score.attrs['raw'] = DataAssembly([0, 2, 1, 3], coords={'a': [1, 1, 2, 2]}, dims=['a']) mean_score = score.mean('a') np.testing.assert_array_equal(mean_score.raw['a'], [1, 1, 2, 2])
def build_response_matrix_from_responses(self, responses): num_choices = [(image_id, choice) for image_id, choice in zip(responses['image_id'].values, responses.values)] num_choices = Counter(num_choices) num_objects = [[(image_id, sample_obj), (image_id, dist_obj)] for image_id, sample_obj, dist_obj in zip( responses['image_id'].values, responses['sample_obj'].values, responses['dist_obj'].values)] num_objects = Counter(itertools.chain(*num_objects)) choices = np.unique(responses) image_ids, indices = np.unique(responses['image_id'], return_index=True) truths = responses['truth'].values[indices] image_dim = responses['image_id'].dims coords = {**{coord: (dims, value) for coord, dims, value in walk_coords(responses)}, **{'choice': ('choice', choices)}} coords = {coord: (dims, value if dims != image_dim else value[indices]) # align image_dim coords with indices for coord, (dims, value) in coords.items()} response_matrix = np.zeros((len(image_ids), len(choices))) for (image_index, image_id), (choice_index, choice) in itertools.product( enumerate(image_ids), enumerate(choices)): if truths[image_index] == choice: # object == choice, ignore p = np.nan else: # divide by number of times where object was one of the two choices (target or distractor) p = (num_choices[(image_id, choice)] / num_objects[(image_id, choice)]) \ if num_objects[(image_id, choice)] > 0 else np.nan response_matrix[image_index, choice_index] = p response_matrix = DataAssembly(response_matrix, coords=coords, dims=responses.dims + ('choice',)) return response_matrix
def test_alignment(self): assembly = NeuroidAssembly( [[1, 2], [1, 2], [4, 3], [4, 3]], coords={ 'image_id': ('presentation', list(range(4))), 'image_meta': ('presentation', list(range(4))), 'neuroid_id': ('neuroid', list(range(2))), 'neuroid_meta': ('neuroid', list(range(2))) }, dims=['presentation', 'neuroid']) matrix = RSA()(assembly) assert np.all(np.diag(matrix) == approx(1., abs=.001)) assert all(matrix.values[np.triu_indices(matrix.shape[0], k=1)] == matrix.values[np.tril_indices(matrix.shape[0], k=-1)] ), "upper and lower triangular need to be equal" expected = DataAssembly( [[1., 1., -1., -1.], [1., 1., -1., -1.], [-1., -1., 1., 1.], [-1., -1., 1., 1.]], coords={ 'image_id': ('presentation', list(range(4))), 'image_meta': ('presentation', list(range(4))) }, dims=['presentation', 'presentation']) np.testing.assert_array_almost_equal( matrix.values, expected.values) # does not take ordering into account
def expand(assembly, target_dims): def strip(coord): stripped_coord = coord if stripped_coord.endswith('_source'): stripped_coord = stripped_coord[:-len('_source')] if stripped_coord.endswith('_target'): stripped_coord = stripped_coord[:-len('_target')] return stripped_coord def reformat_coord_values(coord, dims, values): stripped_coord = strip(coord) if stripped_coord in target_dims and len(values.shape) == 0: values = np.array([values]) dims = [coord] return dims, values coords = { coord: reformat_coord_values(coord, values.dims, values.values) for coord, values in assembly.coords.items() } dim_shapes = OrderedDict((coord, values[1].shape) for coord, values in coords.items() if strip(coord) in target_dims) shape = [_shape for shape in dim_shapes.values() for _shape in shape] # prepare values for broadcasting by adding new dimensions values = assembly.values for _ in range(sum([dim not in assembly.dims for dim in dim_shapes])): values = values[:, np.newaxis] values = np.broadcast_to(values, shape) return DataAssembly(values, coords=coords, dims=list(dim_shapes.keys()))
def test_int_multi_coord(self): d = DataAssembly( [1, 2, 3, 4, 5, 6], coords={ 'a': ('multi_dim', [1, 1, 1, 1, 1, 1]), 'b': ('multi_dim', ['a', 'a', 'a', 'b', 'b', 'b']), 'c': ('multi_dim', ['a', 'b', 'c', 'd', 'e', 'f']) }, dims=['multi_dim']) g = d.multi_groupby(['a', 'b']).mean() assert g.equals( DataAssembly([2., 5.], coords={ 'a': ('multi_dim', [1, 1]), 'b': ('multi_dim', ['a', 'b']) }, dims=['multi_dim']))
def __call__(self, train_source, train_target, test_source, test_target): assert sorted(train_source['image_id'].values) == sorted(train_target['image_id'].values) assert sorted(test_source['image_id'].values) == sorted(test_target['image_id'].values) self.train_source_assemblies.append(train_source) self.train_target_assemblies.append(train_target) self.test_source_assemblies.append(test_source) self.test_target_assemblies.append(test_target) return DataAssembly(0)
def to_xarray(data): presentation_columns = [column for column in data.columns if column not in ['ost-svm', 'ost-logistic']] data = xr.DataArray(np.stack((data['ost-svm'], data['ost-logistic'])), coords={**{column: ('presentation', data[column]) for column in presentation_columns}, **{'decoder': ['svm', 'logistic']}}, dims=['decoder', 'presentation']) data = data.set_index(presentation=presentation_columns) data = DataAssembly(data) return data
def collect_data(data_dir): response_file = h5py.File(data_dir, 'r') texture_modulation_index, absolute_texture_modulation_index, texture_selectivity, noise_selectivity, \ texture_sparseness, noise_sparseness, variance_ratio, sample_variance, family_variance, max_texture, max_noise = \ calculate_texture_properties(response_file, area='v1') # Bins max_texture_bins = np.logspace(-1, 3, 13, base=10) max_noise_bins = np.logspace(-1, 3, 13, base=10) texture_modulation_index_bins = np.linspace(-1, 1, num=25) absolute_texture_modulation_index_bins = np.linspace(0, 1, num=13) texture_selectivity_bins = np.linspace(0, 1, num=11) noise_selectivity_bins = np.linspace(0, 1, num=11) texture_sparseness_bins = np.linspace(0, 1, num=11) noise_sparseness_bins = np.linspace(0, 1, num=11) variance_ratio_bins = np.logspace(-2, 1, num=13) sample_variance_bins = np.linspace(0, 1, num=11) family_variance_bins = np.linspace(0, 1, num=11) # Create DataAssembly with single neuronal properties and bin information assembly = np.concatenate( (texture_modulation_index, absolute_texture_modulation_index, texture_selectivity, noise_selectivity, texture_sparseness, noise_sparseness, variance_ratio, sample_variance, family_variance, max_texture, max_noise), axis=1) assembly = DataAssembly(assembly, coords={ 'neuroid_id': ('neuroid', range(assembly.shape[0])), 'region': ('neuroid', ['V1'] * assembly.shape[0]), 'neuronal_property': PROPERTY_NAMES }, dims=['neuroid', 'neuronal_property']) assembly.attrs['number_of_trials'] = 20 for p in assembly.coords['neuronal_property'].values: assembly.attrs[p + '_bins'] = eval(p + '_bins') return assembly
def devalois1982b_properties(model_identifier, responses, baseline): _assert_grating_activations(responses) radius = np.array(sorted(set(responses.radius.values))) spatial_frequency = np.array( sorted(set(responses.spatial_frequency.values))) orientation = np.array(sorted(set(responses.orientation.values))) phase = np.array(sorted(set(responses.phase.values))) responses = responses.values baseline = baseline.values assert responses.shape[0] == baseline.shape[0] n_neuroids = responses.shape[0] responses = responses.reshape( (n_neuroids, len(radius), len(spatial_frequency), len(orientation), len(phase))) responses_dc = responses.mean(axis=4) - baseline.reshape((-1, 1, 1, 1)) responses_ac = np.absolute(np.fft.fft(responses)) / len(phase) responses_ac = responses_ac[:, :, :, :, 1] * 2 responses = np.zeros( (n_neuroids, len(radius), len(spatial_frequency), len(orientation), 2)) responses[:, :, :, :, 0] = responses_dc responses[:, :, :, :, 1] = responses_ac del responses_ac, responses_dc max_response = responses.reshape((n_neuroids, -1)).max(axis=1, keepdims=True) peak_spatial_frequency = np.zeros((n_neuroids, 1)) for neur in range(n_neuroids): pref_radius, pref_spatial_frequency, pref_orientation, pref_component = \ np.unravel_index(np.argmax(responses[neur, :, :, :, :]), (len(radius), len(spatial_frequency), len(orientation), 2)) spatial_frequency_curve = responses[neur, pref_radius, :, pref_orientation, pref_component] peak_spatial_frequency[neur] = \ calc_spatial_frequency_tuning(spatial_frequency_curve, spatial_frequency, thrsh=0.707, filt_type='smooth', mode='ratio')[1] properties_data = peak_spatial_frequency good_neuroids = max_response > RESPONSE_THRESHOLD properties_data = properties_data[np.argwhere(good_neuroids)[:, 0], :] properties_data = DataAssembly( properties_data, coords={ 'neuroid_id': ('neuroid', range(properties_data.shape[0])), 'region': ('neuroid', ['V1'] * properties_data.shape[0]), 'neuronal_property': PROPERTY_NAMES }, dims=['neuroid', 'neuronal_property']) return properties_data
def collect_data(): # Preferred orientation data preferred_orientation_hist = np.array([110, 83, 100, 92]) preferred_orientation_bins = np.linspace(-22.5, 157.5, 5) preferred_orientation = gen_sample(preferred_orientation_hist, preferred_orientation_bins, scale='linear') # Create DataAssembly with single neuronal properties and bin information assembly = DataAssembly(preferred_orientation, coords={'neuroid_id': ('neuroid', range(preferred_orientation.shape[0])), 'region': ('neuroid', ['V1'] * preferred_orientation.shape[0]), 'neuronal_property': ['preferred_orientation']}, dims=['neuroid', 'neuronal_property']) assembly.attrs['number_of_trials'] = 20 for p in assembly.coords['neuronal_property'].values: assembly.attrs[p+'_bins'] = eval(p+'_bins') return assembly
def test_single_coord(self): d = DataAssembly(data=[[0, 1, 2, 3, 4, 5, 6], [7, 8, 9, 10, 11, 12, 13], [14, 15, 16, 17, 18, 19, 20]], coords={ "greek": ("a", ['alpha', 'beta', 'gamma']), "colors": ("a", ['red', 'green', 'blue']), "compass": ("b", [ 'north', 'south', 'east', 'west', 'northeast', 'southeast', 'southwest' ]), "integer": ("b", [0, 1, 2, 3, 4, 5, 6]), }, dims=("a", "b")) g = d.multi_groupby(['greek']).mean(...) c = DataAssembly( data=[3, 10, 17], coords={'greek': ('greek', ['alpha', 'beta', 'gamma'])}, dims=['greek']) assert g.equals(c)
def test_get_levels(): assy = DataAssembly( data=[[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12], [13, 14, 15], [16, 17, 18]], coords={ 'up': ("a", ['alpha', 'alpha', 'beta', 'beta', 'beta', 'beta']), 'down': ("a", [1, 1, 1, 1, 2, 2]), 'sideways': ('b', ['x', 'y', 'z']) }, dims=['a', 'b']) assert get_levels(assy) == ["up", "down"]
def test_subtract_mean(self): d = DataAssembly( [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], coords={ 'a': ['a', 'b', 'c', 'd'], 'aa': ('a', ['a', 'a', 'b', 'b']), 'b': ['x', 'y', 'z'] }, dims=['a', 'b']) g = d.multi_dim_apply(['aa', 'b'], lambda x, **_: x - x.mean()) assert g.equals( DataAssembly( [[-1.5, -1.5, -1.5], [1.5, 1.5, 1.5], [-1.5, -1.5, -1.5], [1.5, 1.5, 1.5]], coords={ 'a': ['a', 'b', 'c', 'd'], 'aa': ('a', ['a', 'a', 'b', 'b']), 'b': ['x', 'y', 'z'] }, dims=['a', 'b']))