def test_bin_prop_ci(): skip_if_no_external('scipy') n = 100 succ_thresh = np.random.randint(n) acc = 1 - (float(succ_thresh) / n) bl = np.random.random(n) < acc ds = Dataset(bl) m95 = BinomialProportionCI() m50 = BinomialProportionCI(width=0.5) cids = m95(ds) assert_equal(cids.shape, (2, 1)) # accuracy is in the CI maxdist = cids.samples[1, 0] - acc mindist = acc - cids.samples[1, 0] # but allow for numerical uncertainty proportional to the sample size assert_true(maxdist > 0 or maxdist <= 1. / n) assert_true(mindist > 0 or mindist <= 1. / n) # more than one feature ds = Dataset(np.transpose([bl, np.logical_not(bl)])) ci95 = m95(ds) assert_equal(ci95.shape, (2, 2)) # CIs should be inverse assert_array_almost_equal(1 - ci95.samples[0, ::-1], ci95.samples[1]) ci50 = m50(ds) assert_array_almost_equal(1 - ci50.samples[0, ::-1], ci50.samples[1]) # 50% interval is smaller than 95% assert_true(np.all(ci95.samples[0] < ci50.samples[0])) assert_true(np.all(ci95.samples[1] > ci50.samples[1])) assert_equal(list(ci50.sa.ci_boundary), ['lower', 'upper'])
def test_connectivity_hyperalignment(self): skip_if_no_external('scipy') skip_if_no_external('hdf5') # needed for default results backend hdf5 dss_train, dss_test, surface = self.get_testdata() qe = SurfaceQueryEngine(surface, 10, fa_node_key='node_indices') cha = ConnectivityHyperalignment(mask_ids=[0, 3, 6, 9], seed_indices=[0, 3, 6, 9], seed_queryengines=qe, queryengine=qe) mappers = cha(dss_train) aligned_train = [ mapper.forward(ds) for ds, mapper in zip(dss_train, mappers) ] aligned_test = [ mapper.forward(ds) for ds, mapper in zip(dss_test, mappers) ] for ds in aligned_train + aligned_test: zscore(ds, chunks_attr=None) sim_train_before = self.compute_connectivity_profile_similarity( dss_train) sim_train_after = self.compute_connectivity_profile_similarity( aligned_train) sim_test_before = self.compute_connectivity_profile_similarity( dss_test) sim_test_after = self.compute_connectivity_profile_similarity( aligned_test) # ISC should be higher after CHA for both training and testing data self.assertTrue(sim_train_after.mean() > sim_train_before.mean()) self.assertTrue(sim_test_after.mean() > sim_test_before.mean())
def test_bin_prop_ci(): skip_if_no_external('scipy') n = 100 succ_thresh = np.random.randint(n) acc = 1 - (float(succ_thresh) / n) bl = np.random.random(n) < acc ds = Dataset(bl) m95 = BinomialProportionCI() m50 = BinomialProportionCI(width=0.5) cids = m95(ds) assert_equal(cids.shape, (2, 1)) # accuracy is in the CI maxdist = cids.samples[1, 0] - acc mindist = acc - cids.samples[1, 0] # but allow for numerical uncertainty proportional to the sample size assert_true(maxdist > 0 or maxdist <= 1. / n) assert_true(mindist > 0 or mindist <= 1. / n) # more than one feature ds = Dataset(np.transpose([bl, np.logical_not(bl)])) ci95 = m95(ds) assert_equal(ci95.shape, (2, 2)) # CIs should be inverse assert_array_almost_equal(1 - ci95.samples[0, ::-1], ci95.samples[1]) ci50 = m50(ds) assert_array_almost_equal(1 - ci50.samples[0, ::-1], ci50.samples[1]) # 50% interval is smaller than 95% assert_true(np.all(ci95.samples[0] < ci50.samples[0])) assert_true(np.all(ci95.samples[1] > ci50.samples[1])) assert_equal(list(ci50.sa.ci_boundary), ['lower', 'upper'])
def test_cosmo_dataset(fn): skip_if_no_external('scipy', min_version='0.8') mat = _create_small_mat_dataset_dict() ds_mat = cosmo.from_any(mat) savemat(fn, mat) # test Dataset, filename, dict in matlab form, and input from loadmat for input in (ds_mat, fn, mat, loadmat(fn)): # check dataset creation ds = cosmo.from_any(mat) # ensure dataset has expected vlaues assert_array_equal(ds.samples, mat['samples']) _assert_set_equal(ds.sa.keys(), ['chunks', 'labels', 'targets']) _assert_set_equal(ds.sa.keys(), ['chunks', 'labels', 'targets']) _assert_set_equal(ds.a.keys(), ['name']) assert_array_equal(ds.a.name, 'input') assert_array_equal(ds.sa.chunks, [2, 2]) assert_array_equal(ds.sa.targets, [1, 2]) assert_array_equal(ds.sa.labels, ['yin', 'yan']) assert_array_equal(ds.fa.i, [3, 2, 1]) assert_array_equal(ds.fa.j, [1, 2, 2]) # check mapping to matlab format mat_mapped = cosmo.map2cosmo(ds) for m in (mat, mat_mapped): assert_array_equal(ds_mat.samples, m['samples']) _assert_ds_mat_attributes_equal(ds_mat, m)
def test_fmri_to_cosmo(): skip_if_no_external('nibabel') from mvpa2.datasets.mri import fmri_dataset # test exporting an fMRI dataset to CoSMoMVPA pymvpa_ds = fmri_dataset( samples=pathjoin(pymvpa_dataroot, 'example4d.nii.gz'), targets=[1, 2], sprefix='voxel') cosmomvpa_struct = cosmo.map2cosmo(pymvpa_ds) _assert_set_equal(cosmomvpa_struct.keys(), ['a', 'fa', 'sa', 'samples']) a_dict = dict(_obj2tup(cosmomvpa_struct['a'])) mri_keys = ['imgaffine', 'voxel_eldim', 'voxel_dim'] _assert_subset(mri_keys, a_dict.keys()) for k in mri_keys: c_value = a_dict[k] p_value = pymvpa_ds.a[k].value if isinstance(p_value, tuple): c_value = c_value.ravel() p_value = np.asarray(p_value).ravel() assert_array_almost_equal(c_value, p_value) fa_dict = dict(_obj2tup(cosmomvpa_struct['fa'])) fa_keys = ['voxel_indices'] _assert_set_equal(fa_dict.keys(), fa_keys) for k in fa_keys: assert_array_almost_equal(fa_dict[k].T, pymvpa_ds.fa[k].value)
def test_cosmo_queryengine(fn): skip_if_no_external('scipy', min_version='0.8') nbrhood_mat = _create_small_mat_nbrhood_dict() neighbors = nbrhood_mat['neighbors'] savemat(fn, nbrhood_mat) # test dict in matlab form, filename, and through QueryEngine loader for input in (nbrhood_mat, fn, cosmo.CosmoQueryEngine.from_mat(neighbors)): qe = cosmo.from_any(input) assert_array_equal(qe.ids, [0, 1, 2, 3]) for i in qe.ids: nbr_fids_base0 = neighbors[0, i][0] - 1 assert_array_equal(qe.query_byid(i), nbr_fids_base0) _assert_ds_mat_attributes_equal(qe, nbrhood_mat, ('fa', 'a'))
def test_cosmo_queryengine(fn): skip_if_no_external('scipy', min_version='0.8') nbrhood_mat = _create_small_mat_nbrhood_dict() neighbors = nbrhood_mat['neighbors'] savemat(fn, nbrhood_mat) # test dict in matlab form, filename, and through QueryEngine loader for input in (nbrhood_mat, fn, cosmo.CosmoQueryEngine.from_mat(neighbors)): qe = cosmo.from_any(input) assert_array_equal(qe.ids, [0, 1, 2, 3]) for i in qe.ids: nbr_fids_base0 = neighbors[0, i][0] - 1 assert_array_equal(qe.query_byid(i), nbr_fids_base0) _assert_ds_mat_attributes_equal(qe, nbrhood_mat, ('fa', 'a'))
def test_cosmo_io_h5py(fn): skip_if_no_external('h5py') from mvpa2.base.hdf5 import h5save, h5load # Dataset from cosmo ds = cosmo.from_any(_create_small_mat_dataset_dict()) h5save(fn, ds) ds_loaded = h5load(fn) _assert_ds_equal(ds, ds_loaded) # Queryengine qe = cosmo.from_any(_create_small_mat_nbrhood_dict()) h5save(fn, qe) qe_loaded = h5load(fn) assert_array_equal(qe.ids, qe_loaded.ids) _assert_array_collectable_equal(qe.a, qe_loaded.a) _assert_array_collectable_equal(qe.fa, qe_loaded.fa)
def test_cosmo_io_h5py(fn): skip_if_no_external('h5py') from mvpa2.base.hdf5 import h5save, h5load # Dataset from cosmo ds = cosmo.from_any(_create_small_mat_dataset_dict()) h5save(fn, ds) ds_loaded = h5load(fn) _assert_ds_equal(ds, ds_loaded) # Queryengine qe = cosmo.from_any(_create_small_mat_nbrhood_dict()) h5save(fn, qe) qe_loaded = h5load(fn) assert_array_equal(qe.ids, qe_loaded.ids) _assert_array_collectable_equal(qe.a, qe_loaded.a) _assert_array_collectable_equal(qe.fa, qe_loaded.fa)
def test_cosmo_dataset(fn): skip_if_no_external('scipy', min_version='0.8') mat = _create_small_mat_dataset_dict() ds_mat = cosmo.from_any(mat) savemat(fn, mat) # test Dataset, filename, dict in matlab form, and input from loadmat for input in (ds_mat, fn, mat, loadmat(fn)): # check dataset creation ds = cosmo.from_any(mat) # ensure dataset has expected vlaues assert_array_equal(ds.samples, mat['samples']) _assert_set_equal(ds.sa.keys(), ['chunks', 'labels', 'targets']) _assert_set_equal(ds.sa.keys(), ['chunks', 'labels', 'targets']) _assert_set_equal(ds.a.keys(), ['name', 'size']) assert_array_equal(ds.a.name, 'input') assert_array_equal(ds.a.size, [3, 2, 1]) assert_array_equal(ds.sa.chunks, [2, 2]) assert_array_equal(ds.sa.targets, [1, 2]) assert_array_equal(ds.sa.labels, ['yin', 'yan']) assert_array_equal(ds.fa.i, [3, 2, 1]) assert_array_equal(ds.fa.j, [1, 2, 2]) for convert_tuples in (True, False): ds_copy = ds.copy(deep=True) if convert_tuples: # use dataset with tuple data ds_copy.a.size = tuple(ds_copy.a.size) # check mapping to matlab format mat_mapped = cosmo.map2cosmo(ds_copy) for m in (mat, mat_mapped): assert_array_equal(ds_mat.samples, m['samples']) _assert_ds_mat_attributes_equal(ds_mat, m)
def test_connectivity_hyperalignment(self): skip_if_no_external('scipy') skip_if_no_external('hdf5') # needed for default results backend hdf5 dss_train, dss_test, surface = self.get_testdata() qe = SurfaceQueryEngine(surface, 10, fa_node_key='node_indices') cha = ConnectivityHyperalignment( mask_ids=[0, 3, 6, 9], seed_indices=[0, 3, 6, 9], seed_queryengines=qe, queryengine=qe) mappers = cha(dss_train) aligned_train = [mapper.forward(ds) for ds, mapper in zip(dss_train, mappers)] aligned_test = [mapper.forward(ds) for ds, mapper in zip(dss_test, mappers)] for ds in aligned_train + aligned_test: zscore(ds, chunks_attr=None) sim_train_before = self.compute_connectivity_profile_similarity(dss_train) sim_train_after = self.compute_connectivity_profile_similarity(aligned_train) sim_test_before = self.compute_connectivity_profile_similarity(dss_test) sim_test_after = self.compute_connectivity_profile_similarity(aligned_test) # ISC should be higher after CHA for both training and testing data self.assertTrue(sim_train_after.mean() > sim_train_before.mean()) self.assertTrue(sim_test_after.mean() > sim_test_before.mean())
def test_simple_cluster_level_thresholding(): nf = 13 nperms = 100 pthr_feature = 0.5 # just for testing pthr_cluster = 0.5 rand_acc = np.random.normal(size=(nperms, nf)) acc = np.random.normal(size=(1, nf)) # Step 1 is to "fit" "Nonparametrics" per each of the features from mvpa2.clfs.stats import Nonparametric dists = [Nonparametric(samples) for samples in rand_acc.T] # we should be able to assert "p" value for each random sample for each feature rand_acc_p = np.array([dist.rcdf(v) for dist, v in zip(dists, rand_acc.T)]).T rand_acc_p_slow = np.array( [[dist.rcdf(v) for dist, v in zip(dists, sample)] for sample in rand_acc]) assert_array_equal(rand_acc_p_slow, rand_acc_p) assert_equal(rand_acc_p.shape, rand_acc.shape) assert (np.all(rand_acc_p <= 1)) assert (np.all(rand_acc_p > 0)) # 2: apply the same to our acc acc_p = np.array([dist.rcdf(v) for dist, v in zip(dists, acc[0])])[None, :] assert (np.all(acc_p <= 1)) assert (np.all(acc_p > 0)) skip_if_no_external('scipy') # Now we need to do our fancy cluster level madness from mvpa2.algorithms.group_clusterthr import \ get_cluster_sizes, _transform_to_pvals, get_cluster_pvals, \ get_thresholding_map, repeat_cluster_vals rand_acc_p_thr = rand_acc_p < pthr_feature acc_p_thr = acc_p < pthr_feature rand_cluster_sizes = get_cluster_sizes(rand_acc_p_thr) acc_cluster_sizes = get_cluster_sizes(acc_p_thr) # This is how we can compute it within present implementation. # It will be a bit different (since it doesn't account for target value if # I got it right), and would work only for accuracies thr_map = get_thresholding_map(rand_acc, pthr_feature) rand_cluster_sizes_ = get_cluster_sizes(rand_acc > thr_map) acc_cluster_sizes_ = get_cluster_sizes(acc > thr_map) assert_equal(rand_cluster_sizes, rand_cluster_sizes_) assert_equal(acc_cluster_sizes, acc_cluster_sizes_) #print rand_cluster_sizes #print acc_cluster_sizes # That is how it is done in group_clusterthr atm # store cluster size histogram for later p-value evaluation # use a sparse matrix for easy consumption (max dim is the number of # features, i.e. biggest possible cluster) from scipy.sparse import dok_matrix scl = dok_matrix((1, nf + 1), dtype=int) for s in rand_cluster_sizes: scl[0, s] = rand_cluster_sizes[s] test_count_sizes = repeat_cluster_vals(acc_cluster_sizes) test_pvals = _transform_to_pvals(test_count_sizes, scl.astype('float')) # needs conversion to array for comparisons test_pvals = np.asanyarray(test_pvals) # critical cluster_level threshold (without FW correction between clusters) # would be clusters_passed_threshold = test_count_sizes[test_pvals <= pthr_cluster] if len(clusters_passed_threshold): thr_cluster_size = min(clusters_passed_threshold) #print("Min cluster size which passed threshold: %d" % thr_cluster_size) else: #print("No clusters passed threshold") pass #print test_count_sizes, test_pvals acc_cluster_ps = get_cluster_pvals(acc_cluster_sizes, rand_cluster_sizes) for test_pval, test_count_size in zip(test_pvals, test_count_sizes): assert_almost_equal(acc_cluster_ps[test_count_size], test_pval)
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the PyMVPA package for the # copyright and license terms. # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Unit tests for CoSMoMVPA dataset (http://cosmomvpa.org)""" from mvpa2.testing.tools import assert_raises, ok_, assert_true, \ assert_equal, assert_array_equal, with_tempfile from mvpa2.testing import skip_if_no_external skip_if_no_external('scipy') from scipy.io import loadmat, savemat, matlab from mvpa2.datasets import cosmo from mvpa2.measures.base import Measure from mvpa2.datasets.base import Dataset from mvpa2.mappers.fx import mean_feature import numpy as np arr = np.asarray ######################### # helper functions
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # # See COPYING file distributed along with the PyMVPA package for the # copyright and license terms. # ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Unit tests for CoSMoMVPA dataset (http://cosmomvpa.org)""" from mvpa2.testing.tools import assert_raises, ok_, assert_true, \ assert_equal, assert_array_equal, with_tempfile from mvpa2.testing import skip_if_no_external skip_if_no_external('scipy') from scipy.io import loadmat, savemat, matlab from mvpa2.datasets import cosmo from mvpa2.measures.base import Measure from mvpa2.datasets.base import Dataset from mvpa2.mappers.fx import mean_feature import numpy as np arr = np.asarray ######################### # helper functions
def test_simple_cluster_level_thresholding(): nf = 13 nperms = 100 pthr_feature = 0.5 # just for testing pthr_cluster = 0.5 rand_acc = np.random.normal(size=(nperms, nf)) acc = np.random.normal(size=(1, nf)) # Step 1 is to "fit" "Nonparametrics" per each of the features from mvpa2.clfs.stats import Nonparametric dists = [Nonparametric(samples) for samples in rand_acc.T] # we should be able to assert "p" value for each random sample for each feature rand_acc_p = np.array( [dist.rcdf(v) for dist, v in zip(dists, rand_acc.T)] ).T rand_acc_p_slow = np.array([ [dist.rcdf(v) for dist, v in zip(dists, sample)] for sample in rand_acc]) assert_array_equal(rand_acc_p_slow, rand_acc_p) assert_equal(rand_acc_p.shape, rand_acc.shape) assert(np.all(rand_acc_p <= 1)) assert(np.all(rand_acc_p > 0)) # 2: apply the same to our acc acc_p = np.array([dist.rcdf(v) for dist, v in zip(dists, acc[0])])[None, :] assert(np.all(acc_p <= 1)) assert(np.all(acc_p > 0)) skip_if_no_external('scipy') # Now we need to do our fancy cluster level madness from mvpa2.algorithms.group_clusterthr import \ get_cluster_sizes, _transform_to_pvals, get_cluster_pvals, \ get_thresholding_map, repeat_cluster_vals rand_acc_p_thr = rand_acc_p < pthr_feature acc_p_thr = acc_p < pthr_feature rand_cluster_sizes = get_cluster_sizes(rand_acc_p_thr) acc_cluster_sizes = get_cluster_sizes(acc_p_thr) # This is how we can compute it within present implementation. # It will be a bit different (since it doesn't account for target value if # I got it right), and would work only for accuracies thr_map = get_thresholding_map(rand_acc, pthr_feature) rand_cluster_sizes_ = get_cluster_sizes(rand_acc > thr_map) acc_cluster_sizes_ = get_cluster_sizes(acc > thr_map) assert_equal(rand_cluster_sizes, rand_cluster_sizes_) assert_equal(acc_cluster_sizes, acc_cluster_sizes_) #print rand_cluster_sizes #print acc_cluster_sizes # That is how it is done in group_clusterthr atm # store cluster size histogram for later p-value evaluation # use a sparse matrix for easy consumption (max dim is the number of # features, i.e. biggest possible cluster) from scipy.sparse import dok_matrix scl = dok_matrix((1, nf + 1), dtype=int) for s in rand_cluster_sizes: scl[0, s] = rand_cluster_sizes[s] test_count_sizes = repeat_cluster_vals(acc_cluster_sizes) test_pvals = _transform_to_pvals(test_count_sizes, scl.astype('float')) # needs conversion to array for comparisons test_pvals = np.asanyarray(test_pvals) # critical cluster_level threshold (without FW correction between clusters) # would be clusters_passed_threshold = test_count_sizes[test_pvals <= pthr_cluster] if len(clusters_passed_threshold): thr_cluster_size = min(clusters_passed_threshold) #print("Min cluster size which passed threshold: %d" % thr_cluster_size) else: #print("No clusters passed threshold") pass #print test_count_sizes, test_pvals acc_cluster_ps = get_cluster_pvals(acc_cluster_sizes, rand_cluster_sizes) for test_pval, test_count_size in zip(test_pvals, test_count_sizes): assert_almost_equal(acc_cluster_ps[test_count_size], test_pval)
def test_to_df_smoke(self, ds): skip_if_no_external('pandas') df = ds.to_df() print(df)