def cluster_sanity(sres):
def clusters_intersect(c1, c2):
s1 = set( c1.voxels )
s2 = set( c2.voxels )
return len(s1.intersection(s2)) > 0
mn_pt = 1e10
mx_nt = -1e10
g, m = calc_grid_and_map(sres.vox_idx)
img = np.zeros(g)
for i, clist in enumerate((sres.ptail_clusters, sres.ntail_clusters)):
for t in xrange(sres.t.shape[1]):
for f in xrange(sres.t.shape[2]):
np.put(img, m, sres.t[:,t,f])
c_tf = clist[t][f]
for c in c_tf:
cvals = np.take(img, c.voxels)
if i==1 and cvals.max() > mx_nt:
mx_nt = cvals.max()
if i==0 and cvals.min() < mn_pt:
mn_pt = cvals.min()
if len(c_tf) > 1:
for c1, c2 in zip(c_tf[:-1], c_tf[1:]):
assert not clusters_intersect(c1, c2), \
'Cluster intersection at tf=(%d,%d)'%(t,f)
print 'estimated ntail cutoff: %1.3f, estimated ptail cutoff: %1.3f'%(mx_nt, mn_pt)
def map_of_significant_clusters(self, tail, alpha,
corrected_dims=(), pooled_dims=()):
"""Make a map based on a score that mixes cluster-size
significance and peak cluster value significance. This score
is then compared to an empirical null distribution generated
by permutation testing of the samples, and a map is created
across all (voxels, time, freq) where the cluster scores
are deemed significant compared to `alpha`.
Parameters
----------
tail : str in {'pos', 'neg'}
Return clusters where test statistic if significantly large
or small, respectively
alpha : float, p < alpha significance level
Returns
-------
cluster_map : ndarray
binary map which is non-negative at significant clusters
Notes
-----
This method is based on
`Combining voxel intensity and cluster extent with permutation
test framework`, Hayasaka and Nichols, 2004
"""
pvals = self.pscore_clusters(tail)
clusters = self.ptail_clusters if tail.lower()=='pos' \
else self.ntail_clusters
g, m = calc_grid_and_map(self.vox_idx)
cmap = np.zeros_like(self.t)
img = np.zeros(g)
for t in xrange(cmap.shape[1]):
for f in xrange(cmap.shape[2]):
scores = pvals[t][f]
c_tf = clusters[t][f]
for p, ci in zip(scores, c_tf):
# if the score beats alpha, then find the
if p < alpha:
# this is actually faster than looking up
# where ci.voxels intersect with map "m"
img.fill(0)
np.put(img, ci.voxels, 1)
cmap[:,t,f] += np.take(img, m)
return cmap
def map_of_clusters_and_labels(stats, tail, tf, pcrit):
"""
Generate a statistics map and a list of clusters based on the
presence of clusters that exceed a critical p-value.
Parameters
----------
stats: a TimeFreqSnPMClusters object
tail: str {'pos', 'neg'}
tf: tuple (time,freq) index
pcrit: float in (0.0,1.0)
Returns
-------
stats_image, clusters
The `stats_image` is the image of the statistical test values.
`clusters` is a list of the significant clusters. If there
are no cluster scores exceeding pcrit, then clusters returns as None.
"""
g, m = calc_grid_and_map(stats.vox_idx)
l_img = np.zeros(g)
t_img = np.zeros(g)
t, f = tf
np.put(t_img, m, stats.t[:,t,f])
scores = stats.pscore_clusters(tail)
if tail.lower() == 'pos':
print 'doing pos tail'
clusters = stats.ptail_clusters
else:
print 'doing neg tail'
clusters = stats.ntail_clusters
if not clusters[t][f]:
print 'no clusters!'
return t_img, None, 0
crit_scores = (scores[t][f] < pcrit)
if not crit_scores.any():
print 'no significant clusters!'
return t_img, None, 0
else:
tfclusters = clusters[t][f]
idx = np.argwhere(crit_scores).reshape(-1)
critical_clusters = [clusters[t][f][i] for i in idx]
return t_img, critical_clusters
def quick_plot_top_n(stats, tail, n=3):
scores = stats.pscore_clusters(tail)
clusters = stats.ptail_clusters if tail.lower()=='pos' \
else stats.ntail_clusters
# flatten score into a flat list of cluster scores
flattened_scores = []
# flatten clusters into a ntime x nfreq list of cluster sublists
flattened_clusters = []
for crow, srow in zip(clusters, scores):
flattened_scores = np.r_[flattened_scores,
reduce(lambda x, y: x+y,
[ list(e) for e in srow ])]
flattened_clusters += crow
tf_map = []
cluster_lookup = dict()
nc = 0
for i, cl in enumerate(flattened_clusters):
nt, nf = stats.t.shape[1:]
t, f = i / nf, i % nf
tf_map += [ (t,f) ] * len(cl)
cluster_lookup.update( dict( zip(xrange(nc, nc+len(cl)), cl) ) )
nc += len(cl)
n_idx = flattened_scores.argsort()[:n]
g, m = calc_grid_and_map(stats.vox_idx)
imgs = []
tf_pts = []
nclusters = []
for i in n_idx:
clst = cluster_lookup[i]
tf_pts.append('(t,f) = %s, %d pts, p = %1.2f'%(str(tf_map[i]),
clst.size,
flattened_scores[i]))
nclusters.append(cluster_lookup[i])
t_img = np.zeros(g)
t, f = tf_map[i]
np.put(t_img, m, stats.t[:,t,f])
imgs.append(t_img)
quick_plot_clusters(imgs, nclusters, titles=tf_pts)
def test(self, analyze_clusters=False,
cluster_critical_pval=.005, cluster_connectivity=18):
"""
Runs a univariate statistical test at each time-frequency-voxel,
filling in these measures:
* T -- the statistic at each point
* maxT -- the maximum statistic of all permuted measurements
* minT -- the minimum statistic of all permuted measurements
* percentiles -- the "ranking" of the statistics in T relative to all
the permuted statistics
If analyzing clusters, then also collect the maximum cluster size
distribution across permutations. These are the cluster level
analysis parameters:
Parameters
----------
cluster_critical_pval : float
Use the tvalue at this non-empirical quantile as the
cluster-defining threshold.
cluster_connectivity : int in {6, 18, 26}
The connectivity rule for voxels.
* 6 -- connected if faces touch
* 18 -- connected if faces or edges touch
* 26 -- connected if faces, edges, or corners touch
"""
if not self._is_init:
print 'loading and comparing beams'
self._init_data()
n_perm_tested = self.n_perm/2 if self.half_perms else self.n_perm
beams = self.sample_beams
b = beams[0]
n_vox = len(b.voxels)
n_bands = len(b.bands)
n_times = len(b.timewindow)
vox_size = b.voxelsize
# VERY IMPORTANT! THIS GRID SHAPE AND MAP DETERMINE THE
# CLUSTER VOXELS.. POTENTIALY QUITE BRITTLE!!
grid_shape, flat_map = calc_grid_and_map(b.voxel_indices)
# at each (t, f) pt, want to find:
# T test
# max T stat (max across voxels)
# min T stat (min across voxels)
# percentiles (converted to uncorrected p scores)
T = np.empty( (n_vox, n_times, n_bands) )
maxT = np.empty((self.n_perm, n_times, n_bands))
minT = np.empty((self.n_perm, n_times, n_bands))
percentiles = np.empty_like(T)
if analyze_clusters:
# also need a list of the cluster info from each permutation test
# this will include (size, maximal stat, cluster mass)
all_ptail_clusters = []
pc_nulls = np.empty_like(maxT)
all_ntail_clusters = []
nc_nulls = np.empty_like(maxT)
# divide pval by 2, since we're testing both tails
tc = dist.t.isf(cluster_critical_pval, self.dm_gen.dof)
else:
tc = None
# the array of test results
tt = np.empty((self.n_perm, n_vox))
# smoothing kernel size in pixels
fwhm = np.array([20.,20.,20])/np.asarray(vox_size)
for t in xrange(n_times):
for f in xrange(n_bands):
print 'performing stat test at (t,f) = (',t,f,')'
X = np.array([b.s[:,t,f] for b in beams])
p_res = snpm.run_snpm_tests(
X, self.dm_gen, self.beta_weights, n_perm_tested,
symmetry=self.half_perms,
smooth_variance=self.smoothed_variance,
analyze_clusters=analyze_clusters,
grid_shape=grid_shape, vox_map=flat_map,
fwhm_pix=fwhm, t_crit=tc,
connectivity=cluster_connectivity,
fill_value=0, out=tt
)
self.dm_gen.reset()
true_t = tt[0]
# get T distributions
maxT[:,t,f] = tt.max(axis=1)
minT[:,t,f] = tt.min(axis=1)
tt = np.sort(tt, axis=0)
# The 1st column holds the number of test values
# smaller than true_t.
# The 2nd column is the voxel index.
locs = np.argwhere(true_t==tt)
vloc = locs[:,1]
percentiles[vloc,t,f] = locs[:,0]/float(self.n_perm)
T[:,t,f] = true_t
if analyze_clusters:
print 'scoring clusters'
pclusts, nclusts = p_res[1]
# Score and record the pos tail clusters
pscores, pc_nulls[:,t,f] = snpm.score_clusters(
maxT[:,t,f], pclusts
)
scored_pclusts = [
ScoredStatCluster.from_cluster(ci, wscore)
for ci, wscore in zip(pclusts[0], pscores)
]
# Score and record the neg tail clusters
nscores, nc_nulls[:,t,f] = snpm.score_clusters(
minT[:,t,f], nclusts
)
scored_nclusts = [
ScoredStatCluster.from_cluster(ci, wscore)
for ci, wscore in zip(nclusts[0], nscores)
]
all_ptail_clusters.append(scored_pclusts)
all_ntail_clusters.append(scored_nclusts)
stats_args = (T, self.avg_beams[0].voxel_indices,
percentiles, maxT, minT)
if analyze_clusters:
stats_args = stats_args + \
(all_ptail_clusters, pc_nulls,
all_ntail_clusters, nc_nulls)
return TimeFreqSnPMClusters(*stats_args)
else:
return TimeFreqSnPMResults(*stats_args)
