def cluster_sanity(sres): def clusters_intersect(c1, c2): s1 = set( c1.voxels ) s2 = set( c2.voxels ) return len(s1.intersection(s2)) > 0 mn_pt = 1e10 mx_nt = -1e10 g, m = calc_grid_and_map(sres.vox_idx) img = np.zeros(g) for i, clist in enumerate((sres.ptail_clusters, sres.ntail_clusters)): for t in xrange(sres.t.shape[1]): for f in xrange(sres.t.shape[2]): np.put(img, m, sres.t[:,t,f]) c_tf = clist[t][f] for c in c_tf: cvals = np.take(img, c.voxels) if i==1 and cvals.max() > mx_nt: mx_nt = cvals.max() if i==0 and cvals.min() < mn_pt: mn_pt = cvals.min() if len(c_tf) > 1: for c1, c2 in zip(c_tf[:-1], c_tf[1:]): assert not clusters_intersect(c1, c2), \ 'Cluster intersection at tf=(%d,%d)'%(t,f) print 'estimated ntail cutoff: %1.3f, estimated ptail cutoff: %1.3f'%(mx_nt, mn_pt)
def map_of_significant_clusters(self, tail, alpha, corrected_dims=(), pooled_dims=()): """Make a map based on a score that mixes cluster-size significance and peak cluster value significance. This score is then compared to an empirical null distribution generated by permutation testing of the samples, and a map is created across all (voxels, time, freq) where the cluster scores are deemed significant compared to `alpha`. Parameters ---------- tail : str in {'pos', 'neg'} Return clusters where test statistic if significantly large or small, respectively alpha : float, p < alpha significance level Returns ------- cluster_map : ndarray binary map which is non-negative at significant clusters Notes ----- This method is based on `Combining voxel intensity and cluster extent with permutation test framework`, Hayasaka and Nichols, 2004 """ pvals = self.pscore_clusters(tail) clusters = self.ptail_clusters if tail.lower()=='pos' \ else self.ntail_clusters g, m = calc_grid_and_map(self.vox_idx) cmap = np.zeros_like(self.t) img = np.zeros(g) for t in xrange(cmap.shape[1]): for f in xrange(cmap.shape[2]): scores = pvals[t][f] c_tf = clusters[t][f] for p, ci in zip(scores, c_tf): # if the score beats alpha, then find the if p < alpha: # this is actually faster than looking up # where ci.voxels intersect with map "m" img.fill(0) np.put(img, ci.voxels, 1) cmap[:,t,f] += np.take(img, m) return cmap
def map_of_clusters_and_labels(stats, tail, tf, pcrit): """ Generate a statistics map and a list of clusters based on the presence of clusters that exceed a critical p-value. Parameters ---------- stats: a TimeFreqSnPMClusters object tail: str {'pos', 'neg'} tf: tuple (time,freq) index pcrit: float in (0.0,1.0) Returns ------- stats_image, clusters The `stats_image` is the image of the statistical test values. `clusters` is a list of the significant clusters. If there are no cluster scores exceeding pcrit, then clusters returns as None. """ g, m = calc_grid_and_map(stats.vox_idx) l_img = np.zeros(g) t_img = np.zeros(g) t, f = tf np.put(t_img, m, stats.t[:,t,f]) scores = stats.pscore_clusters(tail) if tail.lower() == 'pos': print 'doing pos tail' clusters = stats.ptail_clusters else: print 'doing neg tail' clusters = stats.ntail_clusters if not clusters[t][f]: print 'no clusters!' return t_img, None, 0 crit_scores = (scores[t][f] < pcrit) if not crit_scores.any(): print 'no significant clusters!' return t_img, None, 0 else: tfclusters = clusters[t][f] idx = np.argwhere(crit_scores).reshape(-1) critical_clusters = [clusters[t][f][i] for i in idx] return t_img, critical_clusters
def quick_plot_top_n(stats, tail, n=3): scores = stats.pscore_clusters(tail) clusters = stats.ptail_clusters if tail.lower()=='pos' \ else stats.ntail_clusters # flatten score into a flat list of cluster scores flattened_scores = [] # flatten clusters into a ntime x nfreq list of cluster sublists flattened_clusters = [] for crow, srow in zip(clusters, scores): flattened_scores = np.r_[flattened_scores, reduce(lambda x, y: x+y, [ list(e) for e in srow ])] flattened_clusters += crow tf_map = [] cluster_lookup = dict() nc = 0 for i, cl in enumerate(flattened_clusters): nt, nf = stats.t.shape[1:] t, f = i / nf, i % nf tf_map += [ (t,f) ] * len(cl) cluster_lookup.update( dict( zip(xrange(nc, nc+len(cl)), cl) ) ) nc += len(cl) n_idx = flattened_scores.argsort()[:n] g, m = calc_grid_and_map(stats.vox_idx) imgs = [] tf_pts = [] nclusters = [] for i in n_idx: clst = cluster_lookup[i] tf_pts.append('(t,f) = %s, %d pts, p = %1.2f'%(str(tf_map[i]), clst.size, flattened_scores[i])) nclusters.append(cluster_lookup[i]) t_img = np.zeros(g) t, f = tf_map[i] np.put(t_img, m, stats.t[:,t,f]) imgs.append(t_img) quick_plot_clusters(imgs, nclusters, titles=tf_pts)
def test(self, analyze_clusters=False, cluster_critical_pval=.005, cluster_connectivity=18): """ Runs a univariate statistical test at each time-frequency-voxel, filling in these measures: * T -- the statistic at each point * maxT -- the maximum statistic of all permuted measurements * minT -- the minimum statistic of all permuted measurements * percentiles -- the "ranking" of the statistics in T relative to all the permuted statistics If analyzing clusters, then also collect the maximum cluster size distribution across permutations. These are the cluster level analysis parameters: Parameters ---------- cluster_critical_pval : float Use the tvalue at this non-empirical quantile as the cluster-defining threshold. cluster_connectivity : int in {6, 18, 26} The connectivity rule for voxels. * 6 -- connected if faces touch * 18 -- connected if faces or edges touch * 26 -- connected if faces, edges, or corners touch """ if not self._is_init: print 'loading and comparing beams' self._init_data() n_perm_tested = self.n_perm/2 if self.half_perms else self.n_perm beams = self.sample_beams b = beams[0] n_vox = len(b.voxels) n_bands = len(b.bands) n_times = len(b.timewindow) vox_size = b.voxelsize # VERY IMPORTANT! THIS GRID SHAPE AND MAP DETERMINE THE # CLUSTER VOXELS.. POTENTIALY QUITE BRITTLE!! grid_shape, flat_map = calc_grid_and_map(b.voxel_indices) # at each (t, f) pt, want to find: # T test # max T stat (max across voxels) # min T stat (min across voxels) # percentiles (converted to uncorrected p scores) T = np.empty( (n_vox, n_times, n_bands) ) maxT = np.empty((self.n_perm, n_times, n_bands)) minT = np.empty((self.n_perm, n_times, n_bands)) percentiles = np.empty_like(T) if analyze_clusters: # also need a list of the cluster info from each permutation test # this will include (size, maximal stat, cluster mass) all_ptail_clusters = [] pc_nulls = np.empty_like(maxT) all_ntail_clusters = [] nc_nulls = np.empty_like(maxT) # divide pval by 2, since we're testing both tails tc = dist.t.isf(cluster_critical_pval, self.dm_gen.dof) else: tc = None # the array of test results tt = np.empty((self.n_perm, n_vox)) # smoothing kernel size in pixels fwhm = np.array([20.,20.,20])/np.asarray(vox_size) for t in xrange(n_times): for f in xrange(n_bands): print 'performing stat test at (t,f) = (',t,f,')' X = np.array([b.s[:,t,f] for b in beams]) p_res = snpm.run_snpm_tests( X, self.dm_gen, self.beta_weights, n_perm_tested, symmetry=self.half_perms, smooth_variance=self.smoothed_variance, analyze_clusters=analyze_clusters, grid_shape=grid_shape, vox_map=flat_map, fwhm_pix=fwhm, t_crit=tc, connectivity=cluster_connectivity, fill_value=0, out=tt ) self.dm_gen.reset() true_t = tt[0] # get T distributions maxT[:,t,f] = tt.max(axis=1) minT[:,t,f] = tt.min(axis=1) tt = np.sort(tt, axis=0) # The 1st column holds the number of test values # smaller than true_t. # The 2nd column is the voxel index. locs = np.argwhere(true_t==tt) vloc = locs[:,1] percentiles[vloc,t,f] = locs[:,0]/float(self.n_perm) T[:,t,f] = true_t if analyze_clusters: print 'scoring clusters' pclusts, nclusts = p_res[1] # Score and record the pos tail clusters pscores, pc_nulls[:,t,f] = snpm.score_clusters( maxT[:,t,f], pclusts ) scored_pclusts = [ ScoredStatCluster.from_cluster(ci, wscore) for ci, wscore in zip(pclusts[0], pscores) ] # Score and record the neg tail clusters nscores, nc_nulls[:,t,f] = snpm.score_clusters( minT[:,t,f], nclusts ) scored_nclusts = [ ScoredStatCluster.from_cluster(ci, wscore) for ci, wscore in zip(nclusts[0], nscores) ] all_ptail_clusters.append(scored_pclusts) all_ntail_clusters.append(scored_nclusts) stats_args = (T, self.avg_beams[0].voxel_indices, percentiles, maxT, minT) if analyze_clusters: stats_args = stats_args + \ (all_ptail_clusters, pc_nulls, all_ntail_clusters, nc_nulls) return TimeFreqSnPMClusters(*stats_args) else: return TimeFreqSnPMResults(*stats_args)