def test_tsboxplot():
skip_if_no_external('scipy')
skip_if_no_external('numpy',
min_version='1.5') # for .percentile. approx version
ts = list(range(5))
assert_raises(ValueError, compute_ts_boxplot_stats, ts)
assert_raises(ValueError, compute_ts_boxplot_stats, [ts])
stats, outlier = compute_ts_boxplot_stats(([ts, ts]))
assert_true(len(stats) > 5) # we have some stats
# the are all the same length and match the input timeseries
for k in stats:
assert_equal(len(stats[k]), len(ts))
# outlier array matches the shape and actually contains no
# outliers
assert_equal(outlier.shape, (2, len(ts)))
assert_true(np.all(outlier.mask))
assert_array_equal(stats['mean'], stats['median'])
assert_array_equal(stats['std'], [0] * len(ts))
# now get an outlier
stats, outlier = compute_ts_boxplot_stats(([ts, ts, ts, [0, 1, 2, 3, 30]]),
outlier_thresh=1.0,
greedy_outlier=True)
# the std should still be all-zero, because the outlier removal kills
# all variance across time series
assert_array_equal(stats['std'], [0] * len(ts))
# the entire last time series should be marked as an outlier
assert_array_equal(np.sum(outlier.mask, axis=1),
[len(ts), len(ts), len(ts), 0])
# now for non-greedy outlier marking
stats, outlier = compute_ts_boxplot_stats(([ts, ts, ts, [0, 1, 2, 3, 30]]),
outlier_thresh=1.0,
greedy_outlier=False)
# only one element is an outlier
assert_equal(outlier.mask.sum(), np.prod(outlier.shape) - 1)
assert_true(outlier.mask[3, -1] == False)
# make sure it can deal with NaN in the orginal data
stats, outlier = compute_ts_boxplot_stats(
([ts, ts, [0, np.nan, 2, 3, 30], ts]),
outlier_thresh=1.0,
greedy_outlier=False)
# NaN must not show up as an outlier
assert_equal(outlier.mask.sum(), np.prod(outlier.shape) - 1)
assert_true(outlier.mask[2, -1] == False)
def test_tsboxplot():
skip_if_no_external('scipy')
skip_if_no_external('numpy', min_version='1.5') # for .percentile. approx version
ts = range(5)
assert_raises(ValueError, compute_ts_boxplot_stats, ts)
assert_raises(ValueError, compute_ts_boxplot_stats, [ts])
stats, outlier = compute_ts_boxplot_stats(([ts, ts]))
assert_true(len(stats) > 5) # we have some stats
# the are all the same length and match the input timeseries
for k in stats:
assert_equal(len(stats[k]), len(ts))
# outlier array matches the shape and actually contains no
# outliers
assert_equal(outlier.shape, (2, len(ts)))
assert_true(np.all(outlier.mask))
assert_array_equal(stats['mean'], stats['median'])
assert_array_equal(stats['std'], [0] * len(ts))
# now get an outlier
stats, outlier = compute_ts_boxplot_stats(([ts, ts, ts, [0,1,2,3,30]]),
outlier_thresh=1.0, greedy_outlier=True)
# the std should still be all-zero, because the outlier removal kills
# all variance across time series
assert_array_equal(stats['std'], [0] * len(ts))
# the entire last time series should be marked as an outlier
assert_array_equal(np.sum(outlier.mask, axis=1),
[len(ts), len(ts), len(ts), 0])
# now for non-greedy outlier marking
stats, outlier = compute_ts_boxplot_stats(([ts, ts, ts, [0,1,2,3,30]]),
outlier_thresh=1.0, greedy_outlier=False)
# only one element is an outlier
assert_equal(outlier.mask.sum(), np.prod(outlier.shape) - 1)
assert_true(outlier.mask[3,-1] == False)
# make sure it can deal with NaN in the orginal data
stats, outlier = compute_ts_boxplot_stats(([ts, ts, [0,np.nan,2,3,30], ts]),
outlier_thresh=1.0, greedy_outlier=False)
# NaN must not show up as an outlier
assert_equal(outlier.mask.sum(), np.prod(outlier.shape) - 1)
assert_true(outlier.mask[2,-1] == False)
def motionqc_plot(data, outlier_abs_minthresh=None, outlier_stdthresh=None, ylabel=None):
import pylab as pl
from mvpa2.misc.plot import timeseries_boxplot, concat_ts_boxplot_stats
from mvpa2.misc.stats import compute_ts_boxplot_stats
# segments x [subjects x timepoints x props]
segment_sizes = [d.shape[1] for d in data]
# get stats for all segments and concatenate them
stats = concat_ts_boxplot_stats(
[compute_ts_boxplot_stats(
d,
outlier_abs_minthresh=outlier_abs_minthresh,
outlier_thresh=outlier_stdthresh,
aggfx=np.linalg.norm,
greedy_outlier=True)
for d in data])
outlier = None
if outlier_stdthresh:
outlier = [list(np.where(np.sum(np.logical_not(o.mask), axis=0))[0])
for o in stats[1]]
# plot
timeseries_boxplot(
stats[0]['median'],
mean=stats[0]['mean'],
std=stats[0]['std'],
n=stats[0]['n'],
min=stats[0]['min'],
max=stats[0]['max'],
p25=stats[0]['p25'],
p75=stats[0]['p75'],
outlierd=stats[1],
segment_sizes=segment_sizes)
xp, xl = pl.xticks()
pl.xticks(xp, ['' for i in xl])
pl.xlim((0, len(stats[0]['n'])))
if ylabel:
pl.ylabel(ylabel)
pl.xlabel('time')
return outlier
def run(args):
import numpy as np
import pylab as pl
from mvpa2.datasets.sources.openfmri import OpenFMRIDataset
from mvpa2.misc.plot import timeseries_boxplot, concat_ts_boxplot_stats
from mvpa2.misc.stats import compute_ts_boxplot_stats
from mvpa2.base import verbose
of = OpenFMRIDataset(args.path)
data = of.get_task_bold_attributes(args.task,
args.estimate_fname,
np.loadtxt,
exclude_subjs=args.exclude_subjs)
segment_sizes = [len(d[d.keys()[0]]) for d in data]
if args.relative is not None:
# recode per-subject estimates wrt their particular reference
ref = {
subj: d[args.relative[1]]
for subj, d in data[args.relative[0]].iteritems()
}
for d in data:
for subj in d:
if subj in d:
d[subj] -= ref[subj]
print subj, d[subj].mean()
# collapse all data into a per-run (subj x vol x estimate) array
data = [np.array(d.values()) for d in data]
# figure setup
pl.figure(figsize=(12, 5))
ax = pl.subplot(211)
plt_props = {
'translation': 'estimate L2-norm in mm',
'rotation': 'estimate L2-norm in deg'
}
def bxplot(stats, label):
stats = concat_ts_boxplot_stats(stats)
# XXX need some way to expose whether there were missing subjects and
# report proper IDs -- for now resort to whining
verbose(
0,
"List of outlier time series follows (if any) [note, subject IDs are enumarations and may differ from dataset subject IDs in case of missing subjects]"
)
for i, run in enumerate([
np.where(np.sum(np.logical_not(o.mask), axis=0))
for o in stats[1]
]):
sids = run[0]
if len(sids):
verbose(0,
"%s r%.3i: %s" % (label, i + 1, [s + 1 for s in sids]))
timeseries_boxplot(stats[0]['median'],
mean=stats[0]['mean'],
std=stats[0]['std'],
n=stats[0]['n'],
min=stats[0]['min'],
max=stats[0]['max'],
p25=stats[0]['p25'],
p75=stats[0]['p75'],
outlierd=stats[1],
segment_sizes=segment_sizes)
pl.title(label)
xp, xl = pl.xticks()
pl.xticks(xp, ['' for i in xl])
pl.xlim((0, len(stats[0]['n'])))
pl.ylabel(plt_props[label])
if args.rad2deg and args.estimate_order == 'rottrans':
convfunc = np.rad2deg
else:
convfunc = lambda x: x
# first three columns
run_stats = [
compute_ts_boxplot_stats(convfunc(d[..., :3]),
outlier_abs_minthresh=args.outlier_minthresh,
outlier_thresh=args.outlier_stdthresh,
aggfx=np.linalg.norm,
greedy_outlier=True) for d in data
]
ax = pl.subplot(211)
if args.estimate_order == 'transrot':
bxplot(run_stats, 'translation')
else:
bxplot(run_stats, 'rotation')
# last three columns
if args.rad2deg and args.estimate_order == 'transrot':
convfunc = np.rad2deg
else:
convfunc = lambda x: x
run_stats = [
compute_ts_boxplot_stats(convfunc(d[..., 3:]),
outlier_abs_minthresh=args.outlier_minthresh,
outlier_thresh=args.outlier_stdthresh,
aggfx=np.linalg.norm,
greedy_outlier=True) for d in data
]
ax = pl.subplot(212)
if args.estimate_order == 'rottrans':
bxplot(run_stats, 'translation')
else:
bxplot(run_stats, 'rotation')
pl.xlabel('time in fMRI volumes')
if args.savefig is None:
pl.show()
else:
pl.savefig(args.savefig)
def run(args):
import numpy as np
import pylab as pl
from mvpa2.datasets.sources.openfmri import OpenFMRIDataset
from mvpa2.misc.plot import timeseries_boxplot, concat_ts_boxplot_stats
from mvpa2.misc.stats import compute_ts_boxplot_stats
from mvpa2.base import verbose
of = OpenFMRIDataset(args.path)
data = of.get_task_bold_attributes(
args.task, args.estimate_fname, np.loadtxt,
exclude_subjs=args.exclude_subjs)
segment_sizes = [len(d[d.keys()[0]]) for d in data]
if not args.relative is None:
# recode per-subject estimates wrt their particular reference
ref = {subj: d[args.relative[1]]
for subj, d in data[args.relative[0]].iteritems()}
for d in data:
for subj in d:
if subj in d:
d[subj] -= ref[subj]
print subj, d[subj].mean()
# collapse all data into a per-run (subj x vol x estimate) array
data = [np.array(d.values()) for d in data]
# figure setup
pl.figure(figsize=(12, 5))
ax = pl.subplot(211)
plt_props = {
'translation': 'estimate L2-norm in mm',
'rotation': 'estimate L2-norm in deg'
}
def bxplot(stats, label):
stats = concat_ts_boxplot_stats(stats)
# XXX need some way to expose whether there were missing subjects and
# report proper IDs -- for now resort to whining
verbose(0, "List of outlier time series follows (if any) [note, subject IDs are enumarations and may differ from dataset subject IDs in case of missing subjects]")
for i, run in enumerate([np.where(np.sum(np.logical_not(o.mask), axis=0))
for o in stats[1]]):
sids = run[0]
if len(sids):
verbose(0, "%s r%.3i: %s"
% (label, i + 1, [s + 1 for s in sids]))
timeseries_boxplot(stats[0]['median'],
mean=stats[0]['mean'], std=stats[0]['std'], n=stats[0]['n'],
min=stats[0]['min'], max=stats[0]['max'],
p25=stats[0]['p25'], p75=stats[0]['p75'],
outlierd=stats[1], segment_sizes=segment_sizes)
pl.title(label)
xp, xl = pl.xticks()
pl.xticks(xp, ['' for i in xl])
pl.xlim((0, len(stats[0]['n'])))
pl.ylabel(plt_props[label])
if args.rad2deg and args.estimate_order == 'rottrans':
convfunc = np.rad2deg
else:
convfunc = lambda x: x
# first three columns
run_stats = [compute_ts_boxplot_stats(
convfunc(d[...,:3]),
outlier_abs_minthresh=args.outlier_minthresh,
outlier_thresh=args.outlier_stdthresh,
aggfx=np.linalg.norm,
greedy_outlier=True)
for d in data]
ax = pl.subplot(211)
if args.estimate_order == 'transrot':
bxplot(run_stats, 'translation')
else:
bxplot(run_stats, 'rotation')
# last three columns
if args.rad2deg and args.estimate_order == 'transrot':
convfunc = np.rad2deg
else:
convfunc = lambda x: x
run_stats = [compute_ts_boxplot_stats(
convfunc(d[...,3:]),
outlier_abs_minthresh=args.outlier_minthresh,
outlier_thresh=args.outlier_stdthresh,
aggfx=np.linalg.norm,
greedy_outlier=True)
for d in data]
ax = pl.subplot(212)
if args.estimate_order == 'rottrans':
bxplot(run_stats, 'translation')
else:
bxplot(run_stats, 'rotation')
pl.xlabel('time in fMRI volumes')
if args.savefig is None:
pl.show()
else:
pl.savefig(args.savefig)
def run(args):
import numpy as np
import pylab as pl
from mvpa2.datasets.sources.openfmri import OpenFMRIDataset
from mvpa2.misc.plot import timeseries_boxplot, concat_ts_boxplot_stats
from mvpa2.misc.stats import compute_ts_boxplot_stats
of = OpenFMRIDataset(args.path)
data = of.get_task_bold_attributes(
args.task, args.estimate_fname, np.loadtxt,
exclude_subjs=args.exclude_subjs)
segment_sizes = [len(d[0]) for d in data]
# figure setup
pl.figure(figsize=(12, 5))
ax = pl.subplot(211)
# translation
run_stats = [compute_ts_boxplot_stats(
d[...,:3],
outlier_abs_minthresh=args.outlier_minthresh,
outlier_thresh=args.outlier_stdthresh,
aggfx=np.linalg.norm,
greedy_outlier=True)
for d in data]
stats = concat_ts_boxplot_stats(run_stats)
timeseries_boxplot(stats[0]['median'],
mean=stats[0]['mean'], std=stats[0]['std'], n=stats[0]['n'],
min=stats[0]['min'], max=stats[0]['max'],
p25=stats[0]['p25'], p75=stats[0]['p75'],
outlierd=stats[1], segment_sizes=segment_sizes)
pl.title('translation')
xp, xl = pl.xticks()
pl.xticks(xp, ['' for i in xl])
pl.xlim((0, len(stats[0]['n'])))
#pl.ylim((0,7))
pl.ylabel('estimate L2-norm in mm')
# rotation
ax = pl.subplot(212)
run_stats = [compute_ts_boxplot_stats(
d[...,3:],
outlier_abs_minthresh=args.outlier_minthresh,
outlier_thresh=args.outlier_stdthresh,
aggfx=np.linalg.norm,
greedy_outlier=True)
for d in data]
stats = concat_ts_boxplot_stats(run_stats)
timeseries_boxplot(stats[0]['median'],
mean=stats[0]['mean'], std=stats[0]['std'], n=stats[0]['n'],
min=stats[0]['min'], max=stats[0]['max'],
p25=stats[0]['p25'], p75=stats[0]['p75'],
outlierd=stats[1], segment_sizes=segment_sizes)
pl.xlim((0, len(stats[0]['n'])))
#pl.ylim((0,5))
pl.title('rotation')
pl.ylabel('estimate L2-norm in deg')
pl.xlabel('time in fMRI volumes')
if args.savefig is None:
pl.show()
else:
pl.savefig(args.safefig)
