def __init__(self, source):
"""Reader MEG data from texfiles or file-like objects.
Parameters
----------
source : str or file-like
Strings are assumed to be filenames (with `.gz` suffix
compressed), while all other object types are treated as file-like
objects.
"""
self.ntimepoints = None
self.timepoints = None
self.nsamples = None
self.channelids = []
self.data = []
self.samplingrate = None
# open textfiles
if isinstance(source, str):
if source.endswith('.gz'):
externals.exists('gzip', raise_=True)
import gzip
source = gzip.open(source, 'r')
else:
source = open(source, 'r')
# read file
for line in source:
# split ID
colon = line.find(':')
# ignore lines without id
if colon == -1:
continue
id = line[:colon]
data = line[colon+1:].strip()
if id == 'Sample Number':
timepoints = np.fromstring(data, dtype=int, sep='\t')
# one more as it starts with zero
self.ntimepoints = int(timepoints.max()) + 1
self.nsamples = int(len(timepoints) / self.ntimepoints)
elif id == 'Time':
self.timepoints = np.fromstring(data,
dtype=float,
count=self.ntimepoints,
sep='\t')
self.samplingrate = self.ntimepoints \
/ (self.timepoints[-1] - self.timepoints[0])
else:
# load data
self.data.append(
np.fromstring(data, dtype=float, sep='\t').reshape(
self.nsamples, self.ntimepoints))
# store id
self.channelids.append(id)
# reshape data from (channels x samples x timepoints) to
# (samples x chanels x timepoints)
self.data = np.swapaxes(np.array(self.data), 0, 1)
def plot(self, label_index=0):
"""
TODO: make it friendly to labels given by values?
should we also treat labels_map?
"""
externals.exists("pylab", raise_=True)
import pylab as pl
self._compute()
labels = self._labels
# select only rocs for the given label
rocs = self.rocs[label_index]
fig = pl.gcf()
ax = pl.gca()
pl.plot([0, 1], [0, 1], 'k:')
for ROC in rocs:
pl.plot(ROC.fp, ROC.tp, linewidth=1)
pl.axis((0.0, 1.0, 0.0, 1.0))
pl.axis('scaled')
pl.title('Label %s. Mean AUC=%.2f' % (label_index, self.aucs[label_index]))
pl.xlabel('False positive rate')
pl.ylabel('True positive rate')
def _postcall(self, dataset, result):
"""Some postprocessing on the result
"""
self.raw_result = result
if not self.__transformer is None:
if __debug__:
debug("SA_", "Applying transformer %s" % self.__transformer)
result = self.__transformer(result)
# estimate the NULL distribution when functor is given
if not self.__null_dist is None:
if __debug__:
debug("SA_", "Estimating NULL distribution using %s"
% self.__null_dist)
# we need a matching datameasure instance, but we have to disable
# the estimation of the null distribution in that child to prevent
# infinite looping.
measure = copy.copy(self)
measure.__null_dist = None
self.__null_dist.fit(measure, dataset)
if self.states.isEnabled('null_t'):
# get probability under NULL hyp, but also request
# either it belong to the right tail
null_prob, null_right_tail = \
self.__null_dist.p(result, return_tails=True)
self.null_prob = null_prob
externals.exists('scipy', raiseException=True)
from scipy.stats import norm
# TODO: following logic should appear in NullDist,
# not here
tail = self.null_dist.tail
if tail == 'left':
acdf = N.abs(null_prob)
elif tail == 'right':
acdf = 1.0 - N.abs(null_prob)
elif tail in ['any', 'both']:
acdf = 1.0 - N.clip(N.abs(null_prob), 0, 0.5)
else:
raise RuntimeError, 'Unhandled tail %s' % tail
# We need to clip to avoid non-informative inf's ;-)
# that happens due to lack of precision in mantissa
# which is 11 bits in double. We could clip values
# around 0 at as low as 1e-100 (correspond to z~=21),
# but for consistency lets clip at 1e-16 which leads
# to distinguishable value around p=1 and max z=8.2.
# Should be sufficient range of z-values ;-)
clip = 1e-16
null_t = norm.ppf(N.clip(acdf, clip, 1.0 - clip))
null_t[~null_right_tail] *= -1.0 # revert sign for negatives
self.null_t = null_t # store
else:
# get probability of result under NULL hypothesis if available
# and don't request tail information
self.null_prob = self.__null_dist.p(result)
return result
def _data2img(data, hdr=None, imgtype=None):
# input data is t,x,y,z
if externals.exists("nibabel"):
# let's try whether we can get it done with nibabel
import nibabel
if imgtype is None:
# default is NIfTI1
itype = nibabel.Nifti1Image
else:
itype = imgtype
if issubclass(itype, nibabel.spatialimages.SpatialImage) and (hdr is None or hasattr(hdr, "get_data_dtype")):
# we can handle the desired image type and hdr with nibabel
# use of `None` for the affine should cause to pull it from
# the header
return itype(_get_xyzt_shaped(data), None, hdr)
# otherwise continue and see if there is hope ....
if externals.exists("nifti"):
# maybe pynifti can help
import nifti
if imgtype is None:
itype = nifti.NiftiImage
else:
itype = imgtype
if issubclass(itype, nifti.NiftiImage) and (hdr is None or isinstance(hdr, dict)):
# pynifti wants it transposed
return itype(_get_xyzt_shaped(data).T, hdr)
raise RuntimeError(
"Cannot convert data to an MRI image "
"(backends: nibabel(%s), pynifti(%s). Got hdr='%s', "
"imgtype='%s'." % (externals.exists("nibabel"), externals.exists("nifti"), hdr, imgtype)
)
def __init__(self, sd=0, distribution='rdist', fpp=None, nbins=400, **kwargs):
"""L2-Norm the values, convert them to p-values of a given distribution.
Parameters
----------
sd : int
Samples dimension (if len(x.shape)>1) on which to operate
distribution : string
Which distribution to use. Known are: 'rdist' (later normal should
be there as well)
fpp : float
At what p-value (both tails) if not None, to control for false
positives. It would iteratively prune the tails (tentative real positives)
until empirical p-value becomes less or equal to numerical.
nbins : int
Number of bins for the iterative pruning of positives
WARNING: Highly experimental/slow/etc: no theoretical grounds have been
presented in any paper, nor proven
"""
externals.exists('scipy', raise_=True)
ClassWithCollections.__init__(self, **kwargs)
self.sd = sd
if not (distribution in ['rdist']):
raise ValueError, "Actually only rdist supported at the moment" \
" got %s" % distribution
self.distribution = distribution
self.fpp = fpp
self.nbins = nbins
def __init__(self,
sd=0,
distribution='rdist',
fpp=None,
nbins=400,
**kwargs):
"""L2-Norm the values, convert them to p-values of a given distribution.
Parameters
----------
sd : int
Samples dimension (if len(x.shape)>1) on which to operate
distribution : string
Which distribution to use. Known are: 'rdist' (later normal should
be there as well)
fpp : float
At what p-value (both tails) if not None, to control for false
positives. It would iteratively prune the tails (tentative real positives)
until empirical p-value becomes less or equal to numerical.
nbins : int
Number of bins for the iterative pruning of positives
WARNING: Highly experimental/slow/etc: no theoretical grounds have been
presented in any paper, nor proven
"""
externals.exists('scipy', raise_=True)
ClassWithCollections.__init__(self, **kwargs)
self.sd = sd
if not (distribution in ['rdist']):
raise ValueError, "Actually only rdist supported at the moment" \
" got %s" % distribution
self.distribution = distribution
self.fpp = fpp
self.nbins = nbins
def testExternalsCorrect2ndInvocation(self):
# always fails
externals._KNOWN["checker2"] = "raise ImportError"
self.failUnless(not externals.exists("checker2"), msg="Should be False on 1st invocation")
self.failUnless(not externals.exists("checker2"), msg="Should be False on 2nd invocation as well")
externals._KNOWN.pop("checker2")
def plot(self):
"""Plot correlation coefficients
"""
externals.exists('pylab', raise_=True)
import pylab as pl
pl.plot(self['corrcoef'])
pl.title('Auto-correlation of the sequence')
pl.xlabel('Offset')
pl.ylabel('Correlation Coefficient')
pl.show()
def test_externals_correct2nd_invocation(self):
# always fails
externals._KNOWN['checker2'] = 'raise ImportError'
self.failUnless(not externals.exists('checker2'),
msg="Should be False on 1st invocation")
self.failUnless(not externals.exists('checker2'),
msg="Should be False on 2nd invocation as well")
externals._KNOWN.pop('checker2')
def test_externals_no_double_invocation(self):
# no external should be checking twice (unless specified
# explicitely)
class Checker(object):
"""Helper class to increment count of actual checks"""
def __init__(self): self.checked = 0
def check(self): self.checked += 1
checker = Checker()
externals._KNOWN['checker'] = 'checker.check()'
externals.__dict__['checker'] = checker
externals.exists('checker')
self.failUnlessEqual(checker.checked, 1)
externals.exists('checker')
self.failUnlessEqual(checker.checked, 1)
externals.exists('checker', force=True)
self.failUnlessEqual(checker.checked, 2)
externals.exists('checker')
self.failUnlessEqual(checker.checked, 2)
# restore original externals
externals.__dict__.pop('checker')
externals._KNOWN.pop('checker')
def testExternalsNoDoubleInvocation(self):
# no external should be checking twice (unless specified
# explicitely)
class Checker(object):
"""Helper class to increment count of actual checks"""
def __init__(self):
self.checked = 0
def check(self):
self.checked += 1
checker = Checker()
externals._KNOWN["checker"] = "checker.check()"
externals.__dict__["checker"] = checker
externals.exists("checker")
self.failUnlessEqual(checker.checked, 1)
externals.exists("checker")
self.failUnlessEqual(checker.checked, 1)
externals.exists("checker", force=True)
self.failUnlessEqual(checker.checked, 2)
externals.exists("checker")
self.failUnlessEqual(checker.checked, 2)
# restore original externals
externals.__dict__.pop("checker")
externals._KNOWN.pop("checker")
def _postcall(self, dataset, result):
"""Some postprocessing on the result
"""
self.ca.raw_results = result
# post-processing
result = super(Measure, self)._postcall(dataset, result)
if not self.__null_dist is None:
if self.ca.is_enabled('null_t'):
# get probability under NULL hyp, but also request
# either it belong to the right tail
null_prob, null_right_tail = \
self.__null_dist.p(result, return_tails=True)
self.ca.null_prob = null_prob
externals.exists('scipy', raise_=True)
from scipy.stats import norm
# TODO: following logic should appear in NullDist,
# not here
tail = self.null_dist.tail
if tail == 'left':
acdf = np.abs(null_prob.samples)
elif tail == 'right':
acdf = 1.0 - np.abs(null_prob.samples)
elif tail in ['any', 'both']:
acdf = 1.0 - np.clip(np.abs(null_prob.samples), 0, 0.5)
else:
raise RuntimeError, 'Unhandled tail %s' % tail
# We need to clip to avoid non-informative inf's ;-)
# that happens due to lack of precision in mantissa
# which is 11 bits in double. We could clip values
# around 0 at as low as 1e-100 (correspond to z~=21),
# but for consistency lets clip at 1e-16 which leads
# to distinguishable value around p=1 and max z=8.2.
# Should be sufficient range of z-values ;-)
clip = 1e-16
null_t = norm.ppf(np.clip(acdf, clip, 1.0 - clip))
# assure that we deal with arrays:
null_t = np.array(null_t, ndmin=1, copy=False)
null_t[~null_right_tail] *= -1.0 # revert sign for negatives
null_t_ds = null_prob.copy(deep=False)
null_t_ds.samples = null_t
self.ca.null_t = null_t_ds # store as a Dataset
else:
# get probability of result under NULL hypothesis if available
# and don't request tail information
self.ca.null_prob = self.__null_dist.p(result)
return result
def _reverse(self, data):
if __debug__:
debug('MAP', "Converting signal back using DWP")
if self.__level is None:
raise NotImplementedError
else:
if not externals.exists('pywt wp reconstruct'):
raise NotImplementedError, \
"Reconstruction for a single level for versions of " \
"pywt < 0.1.7 (revision 103) is not supported"
if not externals.exists('pywt wp reconstruct fixed'):
warning("Reconstruction using available version of pywt might "
"result in incorrect data in the tails of the signal")
return self.__reverseSingleLevel(data)
def __init__(self,
gnb,
splitter,
qe,
errorfx=MeanMismatchErrorFx(),
indexsum=None,
**kwargs):
"""Initialize a GNBSearchlight
Parameters
----------
gnb : `GNB`
`GNB` classifier as the specification of what GNB parameters
to use. Instance itself isn't used.
splitter : `Splitter`
`Splitter` to use to compute the error.
errorfx : func, optional
Functor that computes a scalar error value from the vectors of
desired and predicted values (e.g. subclass of `ErrorFunction`)
indexsum : ('sparse', 'fancy'), optional
What use to compute sums over arbitrary columns. 'fancy'
corresponds to regular fancy indexing over columns, whenever
in 'sparse', produce of sparse matrices is used (usually
faster, so is default if `scipy` is available.
"""
# init base class first
BaseSearchlight.__init__(self, qe, **kwargs)
self._errorfx = errorfx
self._splitter = splitter
self._gnb = gnb
if indexsum is None:
if externals.exists('scipy'):
indexsum = 'sparse'
else:
indexsum = 'fancy'
else:
if indexsum == 'sparse' and not externals.exists('scipy'):
warning("Scipy.sparse isn't available so taking 'fancy' as "
"'indexsum' method.")
indexsum = 'fancy'
self._indexsum = indexsum
if not self._nproc in (None, 1):
raise NotImplementedError, "For now only nproc=1 (or None for " \
"autodetection) is supported by GNBSearchlight"
def test_chi_square_searchlight(self):
# only do partial to save time
# Can't yet do this since test_searchlight isn't yet "under nose"
#skip_if_no_external('scipy')
if not externals.exists('scipy'):
return
from mvpa.misc.stats import chisquare
transerror = TransferError(sample_clf_lin)
cv = CrossValidatedTransferError(
transerror,
NFoldSplitter(cvtype=1),
enable_ca=['confusion'])
def getconfusion(data):
cv(data)
return chisquare(cv.ca.confusion.matrix)[0]
sl = sphere_searchlight(getconfusion, radius=0,
center_ids=[3,50])
# run searchlight
results = sl(self.dataset)
self.failUnless(results.nfeatures == 2)
def __init__(self, normalizer_cls=None, normalizer_args=None, **kwargs):
"""
Parameters
----------
normalizer_cls : sg.Kernel.CKernelNormalizer
Class to use as a normalizer for the kernel. Will be instantiated
upon compute(). Only supported for shogun >= 0.6.5.
By default (if left None) assigns IdentityKernelNormalizer to assure no
normalization.
normalizer_args : None or list
If necessary, provide a list of arguments for the normalizer.
"""
SGKernel.__init__(self, **kwargs)
if (normalizer_cls is not None) and (versions['shogun:rev'] < 3377):
raise ValueError, \
"Normalizer specification is supported only for sg >= 0.6.5. " \
"Please upgrade shogun python modular bindings."
if normalizer_cls is None and exists('sg ge 0.6.5'):
normalizer_cls = sgk.IdentityKernelNormalizer
self._normalizer_cls = normalizer_cls
if normalizer_args is None:
normalizer_args = []
self._normalizer_args = normalizer_args
def test_confusion_plot2(self):
array = np.array
uint8 = np.uint8
sets = [(array([1, 2]), array([1, 1]),
array([[0.54343765, 0.45656235], [0.92395853, 0.07604147]])),
(array([1, 2]), array([1, 1]),
array([[0.98030832, 0.01969168], [0.78998763, 0.21001237]])),
(array([1, 2]), array([1, 1]),
array([[0.86125263, 0.13874737], [0.83674113, 0.16325887]])),
(array([1, 2]), array([1, 1]),
array([[0.57870383, 0.42129617], [0.59702509, 0.40297491]])),
(array([1, 2]), array([1, 1]),
array([[0.89530255, 0.10469745], [0.69373919, 0.30626081]])),
(array([1, 2]), array([1, 1]),
array([[0.75015218, 0.24984782], [0.9339767, 0.0660233]])),
(array([1, 2]), array([1, 2]),
array([[0.97826616, 0.02173384], [0.38620638, 0.61379362]])),
(array([2]), array([2]), array([[0.46893776, 0.53106224]]))]
try:
cm = ConfusionMatrix(sets=sets)
except:
self.fail()
if externals.exists("pylab plottable"):
import pylab as pl
#pl.figure()
#print cm
fig, im, cb = cm.plot(origin='lower', numbers=True)
#pl.plot()
self.failUnless((cm._plotted_confusionmatrix == cm.matrix).all())
pl.close(fig)
def testChiSquareSearchlight(self):
# only do partial to save time
if not externals.exists('scipy'):
return
from mvpa.misc.stats import chisquare
transerror = TransferError(sample_clf_lin)
cv = CrossValidatedTransferError(
transerror,
NFoldSplitter(cvtype=1),
enable_states=['confusion'])
def getconfusion(data):
cv(data)
return chisquare(cv.confusion.matrix)[0]
# contruct radius 1 searchlight
sl = Searchlight(getconfusion, radius=1.0,
center_ids=[3,50])
# run searchlight
results = sl(self.dataset)
self.failUnless(len(results) == 2)
def _img2data(src):
# break early of nothing has been given
# XXX feels a little strange to handle this so deep inside, but well...
if src is None:
return None
excpt = None
if externals.exists('nibabel'):
# let's try whether we can get it done with nibabel
import nibabel
if isinstance(src, str):
# filename
try:
img = nibabel.load(src)
except nibabel.spatialimages.ImageFileError, excpt:
# nibabel has some problem, but we might be lucky with
# pynifti below. if not, we have stored the exception
# and raise it below
img = None
pass
else:
# assume this is an image already
img = src
if isinstance(img, nibabel.spatialimages.SpatialImage):
# nibabel image, dissect and return pieces
return _get_txyz_shaped(img.get_data()), img.get_header()
def save(dataset, destination, name=None, compression=None):
"""Save Dataset into HDF5 file
Parameters
----------
dataset : `Dataset`
destination : `h5py.highlevel.File` or str
name : str, optional
compression : None or int or {'gzip', 'szip', 'lzf'}, optional
Level of compression for gzip, or another compression strategy.
"""
if not externals.exists('h5py'):
raise RuntimeError("Missing 'h5py' package -- saving is not possible.")
import h5py
from mvpa.base.hdf5 import obj2hdf
# look if we got an hdf file instance already
if isinstance(destination, h5py.highlevel.File):
own_file = False
hdf = destination
else:
own_file = True
hdf = h5py.File(destination, 'w')
obj2hdf(hdf, dataset, name, compression=compression)
# if we opened the file ourselves we close it now
if own_file:
hdf.close()
return
def testResampling(self):
ds = EEPDataset(os.path.join(pymvpa_dataroot, 'eep.bin'),
labels=[1, 2], labels_map={1:100, 2:101})
channelids = N.array(ds.channelids).copy()
self.failUnless(N.round(ds.samplingrate) == 500.0)
if not externals.exists('scipy'):
return
# should puke when called with nothing
self.failUnlessRaises(ValueError, ds.resample)
# now for real -- should divide nsamples into half
rds = ds.resample(sr=250, inplace=False)
# We should have not changed anything
self.failUnless(N.round(ds.samplingrate) == 500.0)
# by default do 'inplace' resampling
ds.resample(sr=250)
for d in [rds, ds]:
self.failUnless(N.round(d.samplingrate) == 250)
self.failUnless(d.nsamples == 2)
self.failUnless(N.abs((d.dt - 1.0/250)/d.dt)<1e-5)
self.failUnless(N.all(d.channelids == channelids))
# lets now see if we still have a mapper
self.failUnless(d.O.shape == (2, len(channelids), 2))
# and labels_map
self.failUnlessEqual(d.labels_map, {1:100, 2:101})
def test_dist_p_value(self):
"""Basic testing of DistPValue"""
if not externals.exists('scipy'):
return
ndb = 200
ndu = 20
nperd = 2
pthr = 0.05
Nbins = 400
# Lets generate already normed data (on sphere) and add some nonbogus features
datau = (np.random.normal(size=(nperd, ndb)))
dist = np.sqrt((datau * datau).sum(axis=1))
datas = (datau.T / dist.T).T
tn = datax = datas[0, :]
dataxmax = np.max(np.abs(datax))
# now lets add true positive features
tp = [-dataxmax * 1.1] * (ndu/2) + [dataxmax * 1.1] * (ndu/2)
x = np.hstack((datax, tp))
# lets add just pure normal to it
x = np.vstack((x, np.random.normal(size=x.shape))).T
for distPValue in (DistPValue(), DistPValue(fpp=0.05)):
result = distPValue(x)
self.failUnless((result>=0).all)
self.failUnless((result<=1).all)
if cfg.getboolean('tests', 'labile', default='yes'):
self.failUnless(distPValue.ca.positives_recovered[0] > 10)
self.failUnless((np.array(distPValue.ca.positives_recovered) +
np.array(distPValue.ca.nulldist_number) == ndb + ndu).all())
self.failUnlessEqual(distPValue.ca.positives_recovered[1], 0)
def __init__(self, queryengine, roi_ids=None, nproc=None, **kwargs):
"""
Parameters
----------
queryengine : QueryEngine
Engine to use to discover the "neighborhood" of each feature.
See :class:`~mvpa.misc.neighborhood.QueryEngine`.
roi_ids : None or list(int) or str
List of feature ids (not coordinates) the shall serve as ROI seeds
(e.g. sphere centers). Alternatively, this can be the name of a
feature attribute of the input dataset, whose non-zero values
determine the feature ids. By default all features will be used.
nproc : None or int
How many processes to use for computation. Requires `pprocess`
external module. If None -- all available cores will be used.
**kwargs
In addition this class supports all keyword arguments of its
base-class :class:`~mvpa.measures.base.Measure`.
"""
Measure.__init__(self, **kwargs)
if nproc > 1 and not externals.exists('pprocess'):
raise RuntimeError("The 'pprocess' module is required for "
"multiprocess searchlights. Please either "
"install python-pprocess, or reduce `nproc` "
"to 1 (got nproc=%i)" % nproc)
self._queryengine = queryengine
if roi_ids is not None and not isinstance(roi_ids, str) \
and not len(roi_ids):
raise ValueError, \
"Cannot run searchlight on an empty list of roi_ids"
self.__roi_ids = roi_ids
self.nproc = nproc
def test_dist_p_value(self):
"""Basic testing of DistPValue"""
if not externals.exists('scipy'):
return
ndb = 200
ndu = 20
nperd = 2
pthr = 0.05
Nbins = 400
# Lets generate already normed data (on sphere) and add some nonbogus features
datau = (np.random.normal(size=(nperd, ndb)))
dist = np.sqrt((datau * datau).sum(axis=1))
datas = (datau.T / dist.T).T
tn = datax = datas[0, :]
dataxmax = np.max(np.abs(datax))
# now lets add true positive features
tp = [-dataxmax * 1.1] * (ndu/2) + [dataxmax * 1.1] * (ndu/2)
x = np.hstack((datax, tp))
# lets add just pure normal to it
x = np.vstack((x, np.random.normal(size=x.shape))).T
for distPValue in (DistPValue(), DistPValue(fpp=0.05)):
result = distPValue(x)
self.failUnless((result>=0).all)
self.failUnless((result<=1).all)
if cfg.getboolean('tests', 'labile', default='yes'):
self.failUnless(distPValue.ca.positives_recovered[0] > 10)
self.failUnless((np.array(distPValue.ca.positives_recovered) +
np.array(distPValue.ca.nulldist_number) == ndb + ndu).all())
self.failUnless(distPValue.ca.positives_recovered[1] == 0)
def skip_if_no_external(dep, ver_dep=None, min_version=None, max_version=None):
"""Raise SkipTest if external is missing
Parameters
----------
dep : string
Name of the external
ver_dep : string, optional
If for version checking use some different key, e.g. shogun:rev.
If not specified, `dep` will be used.
min_version : None or string or tuple
Minimal required version
max_version : None or string or tuple
Maximal required version
"""
if not externals.exists(dep):
raise SkipTest, \
"External %s is not present thus tests battery skipped" % dep
if ver_dep is None:
ver_dep = dep
if min_version is not None and externals.versions[ver_dep] < min_version:
raise SkipTest, \
"Minimal version %s of %s is required. Present version is %s" \
". Test was skipped." \
% (min_version, ver_dep, externals.versions[ver_dep])
if max_version is not None and externals.versions[ver_dep] > max_version:
raise SkipTest, \
"Maximal version %s of %s is required. Present version is %s" \
". Test was skipped." \
% (min_version, ver_dep, externals.versions[ver_dep])
def skip_if_no_external(dep, ver_dep=None, min_version=None, max_version=None):
"""Raise SkipTest if external is missing
Parameters
----------
dep : string
Name of the external
ver_dep : string, optional
If for version checking use some different key, e.g. shogun:rev.
If not specified, `dep` will be used.
min_version : None or string or tuple
Minimal required version
max_version : None or string or tuple
Maximal required version
"""
if not externals.exists(dep):
raise SkipTest, \
"External %s is not present thus tests battery skipped" % dep
if ver_dep is None:
ver_dep = dep
if min_version is not None and externals.versions[ver_dep] < min_version:
raise SkipTest, \
"Minimal version %s of %s is required. Present version is %s" \
". Test was skipped." \
% (min_version, ver_dep, externals.versions[ver_dep])
if max_version is not None and externals.versions[ver_dep] > max_version:
raise SkipTest, \
"Maximal version %s of %s is required. Present version is %s" \
". Test was skipped." \
% (min_version, ver_dep, externals.versions[ver_dep])
def testBinds(self):
ds = normalFeatureDataset()
ds_data = ds.samples.copy()
ds_chunks = ds.chunks.copy()
self.failUnless(N.all(ds.samples == ds_data)) # sanity check
funcs = ['zscore', 'coarsenChunks']
if externals.exists('scipy'):
funcs.append('detrend')
for f in funcs:
eval('ds.%s()' % f)
self.failUnless(N.any(ds.samples != ds_data) or
N.any(ds.chunks != ds_chunks),
msg="We should have modified original dataset with %s" % f)
ds.samples = ds_data.copy()
ds.chunks = ds_chunks.copy()
# and some which should just return results
for f in ['aggregateFeatures', 'removeInvariantFeatures',
'getSamplesPerChunkLabel']:
res = eval('ds.%s()' % f)
self.failUnless(res is not None,
msg='We should have got result from function %s' % f)
self.failUnless(N.all(ds.samples == ds_data),
msg="Function %s should have not modified original dataset" % f)
def __init__(self, normalizer_cls=None, normalizer_args=None, **kwargs):
"""
Parameters
----------
normalizer_cls : sg.Kernel.CKernelNormalizer
Class to use as a normalizer for the kernel. Will be instantiated
upon compute(). Only supported for shogun >= 0.6.5.
By default (if left None) assigns IdentityKernelNormalizer to assure no
normalization.
normalizer_args : None or list
If necessary, provide a list of arguments for the normalizer.
"""
SGKernel.__init__(self, **kwargs)
if (normalizer_cls is not None) and (versions['shogun:rev'] < 3377):
raise ValueError, \
"Normalizer specification is supported only for sg >= 0.6.5. " \
"Please upgrade shogun python modular bindings."
if normalizer_cls is None and exists('sg ge 0.6.5'):
normalizer_cls = sgk.IdentityKernelNormalizer
self._normalizer_cls = normalizer_cls
if normalizer_args is None:
normalizer_args = []
self._normalizer_args = normalizer_args
def save(dataset, destination, name=None, compression=None):
"""Save Dataset into HDF5 file
Parameters
----------
dataset : `Dataset`
destination : `h5py.highlevel.File` or str
name : str, optional
compression : None or int or {'gzip', 'szip', 'lzf'}, optional
Level of compression for gzip, or another compression strategy.
"""
if not externals.exists('h5py'):
raise RuntimeError("Missing 'h5py' package -- saving is not possible.")
import h5py
from mvpa.base.hdf5 import obj2hdf
# look if we got an hdf file instance already
if isinstance(destination, h5py.highlevel.File):
own_file = False
hdf = destination
else:
own_file = True
hdf = h5py.File(destination, 'w')
obj2hdf(hdf, dataset, name, compression=compression)
# if we opened the file ourselves we close it now
if own_file:
hdf.close()
return
def _call(self, dataset):
"""Perform the ROI search.
"""
# local binding
nproc = self.nproc
if nproc is None and externals.exists('pprocess'):
import pprocess
try:
nproc = pprocess.get_number_of_cores() or 1
except AttributeError:
warning("pprocess version %s has no API to figure out maximal "
"number of cores. Using 1"
% externals.versions['pprocess'])
nproc = 1
# train the queryengine
self._queryengine.train(dataset)
# decide whether to run on all possible center coords or just a provided
# subset
if isinstance(self.__roi_ids, str):
roi_ids = dataset.fa[self.__roi_ids].value.nonzero()[0]
elif self.__roi_ids is not None:
roi_ids = self.__roi_ids
# safeguard against stupidity
if __debug__:
if max(roi_ids) >= dataset.nfeatures:
raise IndexError, \
"Maximal center_id found is %s whenever given " \
"dataset has only %d features" \
% (max(roi_ids), dataset.nfeatures)
else:
roi_ids = np.arange(dataset.nfeatures)
# pass to subclass
results, roi_sizes = self._sl_call(dataset, roi_ids, nproc)
if not roi_sizes is None:
self.ca.roi_sizes = roi_sizes
if 'mapper' in dataset.a:
# since we know the space we can stick the original mapper into the
# results as well
if self.__roi_ids is None:
results.a['mapper'] = copy.copy(dataset.a.mapper)
else:
# there is an additional selection step that needs to be
# expressed by another mapper
mapper = copy.copy(dataset.a.mapper)
mapper.append(StaticFeatureSelection(roi_ids,
dshape=dataset.shape[1:]))
results.a['mapper'] = mapper
# charge state
self.ca.raw_results = results
# return raw results, base-class will take care of transformations
return results
def _call(self, dataset, labels=None):
# This code is based on SciPy's stats.f_oneway()
# Copyright (c) Gary Strangman. All rights reserved
# License: BSD
#
# However, it got tweaked and optimized to better fit into PyMVPA.
# number of groups
if labels is None:
labels = dataset.targets
ul = np.unique(labels)
na = len(ul)
bign = float(dataset.nsamples)
alldata = dataset.samples
# total squares of sums
sostot = np.sum(alldata, axis=0)
sostot *= sostot
sostot /= bign
# total sum of squares
sstot = np.sum(alldata * alldata, axis=0) - sostot
# between group sum of squares
ssbn = 0
for l in ul:
# all samples for the respective label
d = alldata[labels == l]
sos = np.sum(d, axis=0)
sos *= sos
ssbn += sos / float(len(d))
ssbn -= sostot
# within
sswn = sstot - ssbn
# degrees of freedom
dfbn = na-1
dfwn = bign - na
# mean sums of squares
msb = ssbn / float(dfbn)
msw = sswn / float(dfwn)
f = msb / msw
# assure no NaNs -- otherwise it leads instead of
# sane unittest failure (check of NaNs) to crazy
# File "mtrand.pyx", line 1661, in mtrand.shuffle
# TypeError: object of type 'numpy.int64' has no len()
# without any sane backtrace
f[np.isnan(f)] = 0
if externals.exists('scipy'):
from scipy.stats import fprob
return Dataset(f[np.newaxis], fa={'fprob': fprob(dfbn, dfwn, f)})
else:
return Dataset(f[np.newaxis])
def _acquire_externals(self, out):
# Test and list all dependencies:
sdeps = {True: [], False: [], 'Error': []}
for dep in sorted(externals._KNOWN):
try:
sdeps[externals.exists(dep, force=False)] += [dep]
except:
sdeps['Error'] += [dep]
out.write('EXTERNALS:\n')
out.write(' Present: %s\n' % ', '.join(sdeps[True]))
out.write(' Absent: %s\n' % ', '.join(sdeps[False]))
if len(sdeps['Error']):
out.write(' Errors in determining: %s\n' %
', '.join(sdeps['Error']))
SV = ('.__version__', ) # standard versioning
out.write(' Versions of critical externals:\n')
# First the ones known to externals,
# TODO: make all of them set in externals.versions
for k, v in externals.versions.iteritems():
out.write(' %-12s: %s\n' % (k, str(v)))
for e, mname, fs in (
('ctypes', None, SV),
('matplotlib', None, SV),
('lxml', None, ('.etree.__version__', )),
('nifti', None, SV),
('numpy', None, SV),
('openopt', 'openopt', SV),
('openopt', 'scikits.openopt', ('.openopt.__version__', )),
('pywt', None, SV),
#('rpy', None, ('.rpy_version',)),
('shogun', None, ('.Classifier.Version_get_version_release()', )),
):
try:
if not externals.exists(e):
continue #sver = 'not present'
else:
if mname is None:
mname = e
m = __import__(mname)
svers = [eval('m%s' % (f, )) for f in fs]
sver = ' '.join(svers)
except Exception, exc:
sver = 'failed to query due to "%s"' % str(exc)
out.write(' %-12s: %s\n' % (e, sver))
def _wm_reverse(self, data):
if __debug__:
debug('MAP', "Converting signal back using DWP")
if self.__level is None:
raise NotImplementedError
else:
if not externals.exists('pywt wp reconstruct'):
raise NotImplementedError, \
"Reconstruction for a single level for versions of " \
"pywt < 0.1.7 (revision 103) is not supported"
if not externals.exists('pywt wp reconstruct fixed'):
warning("%s: Reverse mapping with this version of 'pywt' might "
"result in incorrect data in the tails of the signal. "
"Please check for an update of 'pywt', or be careful "
"when interpreting the edges of the reverse mapped "
"data." % self.__class__.__name__)
return self.__reverse_single_level(data)
def _call(self, dataset):
# This code is based on SciPy's stats.f_oneway()
# Copyright (c) Gary Strangman. All rights reserved
# License: BSD
#
# However, it got tweaked and optimized to better fit into PyMVPA.
# number of groups
targets_sa = dataset.sa[self._targets_attr]
labels = targets_sa.value
ul = targets_sa.unique
na = len(ul)
bign = float(dataset.nsamples)
alldata = dataset.samples
# total squares of sums
sostot = np.sum(alldata, axis=0)
sostot *= sostot
sostot /= bign
# total sum of squares
sstot = np.sum(alldata * alldata, axis=0) - sostot
# between group sum of squares
ssbn = 0
for l in ul:
# all samples for the respective label
d = alldata[labels == l]
sos = np.sum(d, axis=0)
sos *= sos
ssbn += sos / float(len(d))
ssbn -= sostot
# within
sswn = sstot - ssbn
# degrees of freedom
dfbn = na - 1
dfwn = bign - na
# mean sums of squares
msb = ssbn / float(dfbn)
msw = sswn / float(dfwn)
f = msb / msw
# assure no NaNs -- otherwise it leads instead of
# sane unittest failure (check of NaNs) to crazy
# File "mtrand.pyx", line 1661, in mtrand.shuffle
# TypeError: object of type 'numpy.int64' has no len()
# without any sane backtrace
f[np.isnan(f)] = 0
if externals.exists('scipy'):
from scipy.stats import fprob
return Dataset(f[np.newaxis], fa={'fprob': fprob(dfbn, dfwn, f)})
else:
return Dataset(f[np.newaxis])
def _acquire_externals(self, out):
# Test and list all dependencies:
sdeps = {True: [], False: [], 'Error': []}
for dep in sorted(externals._KNOWN):
try:
sdeps[externals.exists(dep, force=False)] += [dep]
except:
sdeps['Error'] += [dep]
out.write('EXTERNALS:\n')
out.write(' Present: %s\n' % ', '.join(sdeps[True]))
out.write(' Absent: %s\n' % ', '.join(sdeps[False]))
if len(sdeps['Error']):
out.write(' Errors in determining: %s\n' % ', '.join(sdeps['Error']))
SV = ('.__version__', ) # standard versioning
out.write(' Versions of critical externals:\n')
# First the ones known to externals,
# TODO: make all of them set in externals.versions
for k, v in externals.versions.iteritems():
out.write(' %-12s: %s\n' % (k, str(v)))
for e, mname, fs in (
('ctypes', None, SV),
('matplotlib', None, SV),
('lxml', None, ('.etree.__version__',)),
('nifti', None, SV),
('numpy', None, SV),
('openopt', 'openopt', SV),
('openopt', 'scikits.openopt', ('.openopt.__version__',)),
('pywt', None, SV),
#('rpy', None, ('.rpy_version',)),
('shogun', None, ('.Classifier.Version_get_version_release()',)),
):
try:
if not externals.exists(e):
continue #sver = 'not present'
else:
if mname is None:
mname = e
m = __import__(mname)
svers = [eval('m%s' % (f,)) for f in fs]
sver = ' '.join(svers)
except Exception, exc:
sver = 'failed to query due to "%s"' % str(exc)
out.write(' %-12s: %s\n' % (e, sver))
def _wm_reverse(self, data):
if __debug__:
debug('MAP', "Converting signal back using DWP")
if self.__level is None:
raise NotImplementedError
else:
if not externals.exists('pywt wp reconstruct'):
raise NotImplementedError, \
"Reconstruction for a single level for versions of " \
"pywt < 0.1.7 (revision 103) is not supported"
if not externals.exists('pywt wp reconstruct fixed'):
warning(
"%s: Reverse mapping with this version of 'pywt' might "
"result in incorrect data in the tails of the signal. "
"Please check for an update of 'pywt', or be careful "
"when interpreting the edges of the reverse mapped "
"data." % self.__class__.__name__)
return self.__reverse_single_level(data)
def __init__(self, gnb, splitter, qe, errorfx=mean_mismatch_error,
indexsum=None, **kwargs):
"""Initialize a GNBSearchlight
Parameters
----------
gnb : `GNB`
`GNB` classifier as the specification of what GNB parameters
to use. Instance itself isn't used.
splitter : `Splitter`
`Splitter` to use to compute the error.
errorfx : func, optional
Functor that computes a scalar error value from the vectors of
desired and predicted values (e.g. subclass of `ErrorFunction`)
indexsum : ('sparse', 'fancy'), optional
What use to compute sums over arbitrary columns. 'fancy'
corresponds to regular fancy indexing over columns, whenever
in 'sparse', produce of sparse matrices is used (usually
faster, so is default if `scipy` is available.
"""
# init base class first
BaseSearchlight.__init__(self, qe, **kwargs)
self._errorfx = errorfx
self._splitter = splitter
self._gnb = gnb
if indexsum is None:
if externals.exists('scipy'):
indexsum = 'sparse'
else:
indexsum = 'fancy'
else:
if indexsum == 'sparse' and not externals.exists('scipy'):
warning("Scipy.sparse isn't available so taking 'fancy' as "
"'indexsum' method.")
indexsum = 'fancy'
self._indexsum = indexsum
if not self._nproc in (None, 1):
raise NotImplementedError, "For now only nproc=1 (or None for " \
"autodetection) is supported by GNBSearchlight"
def _get_nifti_data(nim):
"""Convenience function to extract the data array from a NiftiImage
This function will make use of advanced features of PyNIfTI to prevent
unnecessary copying if a sufficent version is available.
"""
if externals.exists("nifti ge 0.20090205.1"):
return nim.data
else:
return nim.asarray()
def testSimpleWP1Level(self):
"""
"""
ds = datasets["uni2large"]
d2d = ds.samples
ws = 50 # size of timeline for wavelet
sp = (N.arange(ds.nsamples - ws * 2) + ws)[:4]
# create 3D instance (samples x timepoints x channels)
bcm = BoxcarMapper(sp, ws)
d3d = bcm(d2d)
# use wavelet mapper
wdm = WaveletPacketMapper(level=2, wavelet="sym2")
d3d_wd = wdm(d3d)
# Check dimensionality
d3d_wds, d3ds = d3d_wd.shape, d3d.shape
self.failUnless(len(d3d_wds) == len(d3ds) + 1)
self.failUnless(d3d_wds[1] * d3d_wds[2] >= d3ds[1])
self.failUnless(d3d_wds[0] == d3ds[0])
self.failUnless(d3d_wds[-1] == d3ds[-1])
# print d2d.shape, d3d.shape, d3d_wd.shape
if externals.exists("pywt wp reconstruct"):
# Test reverse -- should be identical
# we can do reverse only for DWT
d3d_rev = wdm.reverse(d3d_wd)
# inverse transform might be not exactly as the
# input... but should be very close ;-)
self.failUnlessEqual(d3d_rev.shape, d3d.shape, msg="Shape should be the same after iDWT")
diff = N.linalg.norm(d3d - d3d_rev)
ornorm = N.linalg.norm(d3d)
if externals.exists("pywt wp reconstruct fixed"):
self.failUnless(diff / ornorm < 1e-10)
else:
self.failUnlessRaises(NotImplementedError, wdm.reverse, d3d_wd)
def _get_data_form_pynifti_img(nim):
"""Convenience function to extract the data array from a NiftiImage
This function will make use of advanced features of PyNIfTI to prevent
unnecessary copying if a sufficent version is available.
"""
if externals.exists('nifti ge 0.20090205.1'):
data = nim.data
else:
data = nim.asarray()
# we want the data to be x,y,z,t
return data.T
def _get_data_form_pynifti_img(nim):
"""Convenience function to extract the data array from a NiftiImage
This function will make use of advanced features of PyNIfTI to prevent
unnecessary copying if a sufficent version is available.
"""
if externals.exists('nifti ge 0.20090205.1'):
data = nim.data
else:
data = nim.asarray()
# we want the data to be x,y,z,t
return data.T
def testEventDatasetExtended(self):
if not externals.exists('nifti'):
return
from mvpa.datasets.nifti import ERNiftiDataset
try:
ds = ERNiftiDataset(
samples=os.path.join(pymvpa_dataroot, 'bold.nii.gz'),
mask=os.path.join(pymvpa_dataroot, 'mask.nii.gz'),
events=[Event(onset=1,duration=5,label=1,chunk=1)],
evconv=True, tr=2.0)
except ValueError, e:
self.fail("Failed to create a simple ERNiftiDataset from a volume"
" with only 1 slice. Exception was:\n %s" % e)
def figures(self, *args, **kwargs):
"""Adds all present figures at once
If called twice, it might add the same figure multiple times,
so make sure to close all previous figures if you use
figures() multiple times
"""
if externals.exists('pylab', raise_=True):
import pylab as pl
figs = pl.matplotlib._pylab_helpers.Gcf.figs
if __debug__ and not self in debug.handlers:
debug('REP', "Saving all %d present figures" % len(figs))
for fid, f in figs.iteritems():
self.figure(f.canvas.figure, *args, **kwargs)
def _data2img(data, hdr=None, imgtype=None):
# input data is t,x,y,z
if externals.exists('nibabel'):
# let's try whether we can get it done with nibabel
import nibabel
if imgtype is None:
# default is NIfTI1
itype = nibabel.Nifti1Image
else:
itype = imgtype
if issubclass(itype, nibabel.spatialimages.SpatialImage) \
and (hdr is None or hasattr(hdr, 'get_data_dtype')):
# we can handle the desired image type and hdr with nibabel
# use of `None` for the affine should cause to pull it from
# the header
return itype(_get_xyzt_shaped(data), None, hdr)
# otherwise continue and see if there is hope ....
if externals.exists('nifti'):
# maybe pynifti can help
import nifti
if imgtype is None:
itype = nifti.NiftiImage
else:
itype = imgtype
if issubclass(itype, nifti.NiftiImage) \
and (hdr is None or isinstance(hdr, dict)):
# pynifti wants it transposed
return itype(_get_xyzt_shaped(data).T, hdr)
raise RuntimeError("Cannot convert data to an MRI image "
"(backends: nibabel(%s), pynifti(%s). Got hdr='%s', "
"imgtype='%s'."
% (externals.exists('nibabel'),
externals.exists('nifti'),
hdr,
imgtype))
def test_simple_wp1_level(self):
"""
"""
ds = datasets['uni2large']
d2d = ds.samples
ws = 50 # size of timeline for wavelet
sp = (np.arange(ds.nsamples - ws * 2) + ws)[:4]
# create 3D instance (samples x timepoints x channels)
bcm = BoxcarMapper(sp, ws)
d3d = bcm(d2d)
# use wavelet mapper
wdm = WaveletPacketMapper(level=2, wavelet='sym2')
d3d_wd = wdm(d3d)
# Check dimensionality
d3d_wds, d3ds = d3d_wd.shape, d3d.shape
self.failUnless(len(d3d_wds) == len(d3ds) + 1)
self.failUnless(d3d_wds[1] * d3d_wds[2] >= d3ds[1])
self.failUnless(d3d_wds[0] == d3ds[0])
self.failUnless(d3d_wds[-1] == d3ds[-1])
#print d2d.shape, d3d.shape, d3d_wd.shape
if externals.exists('pywt wp reconstruct'):
# Test reverse -- should be identical
# we can do reverse only for DWT
d3d_rev = wdm.reverse(d3d_wd)
# inverse transform might be not exactly as the
# input... but should be very close ;-)
self.failUnlessEqual(d3d_rev.shape,
d3d.shape,
msg="Shape should be the same after iDWT")
diff = np.linalg.norm(d3d - d3d_rev)
ornorm = np.linalg.norm(d3d)
skip_if_no_external('pywt wp reconstruct fixed')
self.failUnless(diff / ornorm < 1e-10)
else:
self.failUnlessRaises(NotImplementedError, wdm.reverse, d3d_wd)
def run_tests_using_nose(limit=None, verbosity=1, exit_=False):
"""Run nose-based tests -- really really silly way, just to get started
TODO: just switch to using numpy.testing framework, for that
unittests need to be cleaned and unified first
"""
nosetests = collect_nose_tests(verbosity=verbosity)
if not externals.exists('nose'):
warning("You do not have python-nose installed. Some unittests were "
"skipped: %s" % (', '.join(nosetests)))
return
from nose import main
import nose
import nose.config
tests = collect_unit_tests(verbosity=verbosity) + nosetests
config = nose.config.Config(verbosity=verbosity,
plugins=nose.plugins.DefaultPluginManager())
if limit is None:
# Lets see if we aren't missing any:
if verbosity:
import os, glob
testfiles = glob.glob('%s%stest_*.py' %
(os.path.dirname(__file__), os.path.sep))
not_tested = set([os.path.basename(f) for f in testfiles]) \
- set(['%s.py' % f for f in tests])
if len(not_tested):
print(
"T: Warning -- following test files were found but will "
"not be tested: %s" % ', '.join(not_tested))
config.testNames = ['mvpa.tests.' + nt for nt in tests]
else:
config.testNames = [
'mvpa.tests.' + nt for nt in tests if nt[5:] in limit
]
# run the tests
_ = main(defaultTest=(), config=config, exit=exit_)
def __init__(self, kernel=None, **kwargs):
"""Initialize a GPR regression analysis.
Parameters
----------
kernel : Kernel
a kernel object defining the covariance between instances.
(Defaults to SquaredExponentialKernel if None in arguments)
"""
# init base class first
Classifier.__init__(self, **kwargs)
# It does not make sense to calculate a confusion matrix for a GPR
# XXX it does ;) it will be a RegressionStatistics actually ;-)
# So if someone desires -- let him have it
# self.ca.enable('training_confusion', False)
# set kernel:
if kernel is None:
kernel = SquaredExponentialKernel()
debug(
"GPR",
"No kernel was provided, falling back to default: %s" % kernel)
self.__kernel = kernel
# append proper clf_internal depending on the kernel
# TODO: add "__tags__" to kernels since the check
# below does not scale
if isinstance(kernel, GeneralizedLinearKernel) or \
isinstance(kernel, LinearKernel):
self.__tags__ += ['linear']
else:
self.__tags__ += ['non-linear']
if externals.exists('openopt'):
self.__tags__ += ['has_sensitivity']
# No need to initialize conditional attributes. Unless they got set
# they would raise an exception self.predicted_variances =
# None self.log_marginal_likelihood = None
self._init_internals()
pass
def test_custom_sg(self):
lk = sgK.LinearSGKernel()
cl = sgK.CustomSGKernel(sgK.sgk.LinearKernel)
poly = sgK.PolySGKernel()
poly_params = [('order', 2), ('inhomogenous', True)]
if not exists('sg ge 0.6.5'):
poly_params += [('use_normalization', False)]
custom = sgK.CustomSGKernel(sgK.sgk.PolyKernel,
kernel_params=poly_params)
d = np.random.randn(253, 52)
lk.compute(d)
cl.compute(d)
poly.compute(d)
custom.compute(d)
self.failUnless(np.all(lk.as_np()._k == cl.as_np()._k),
'CustomSGKernel does not agree with Linear')
self.failUnless(np.all(poly.as_np()._k == custom.as_np()._k),
'CustomSGKernel does not agree with Poly')
def _img2data(src):
excpt = None
if externals.exists('nibabel'):
# let's try whether we can get it done with nibabel
import nibabel
if isinstance(src, str):
# filename
try:
img = nibabel.load(src)
except nibabel.spatialimages.ImageFileError, excpt:
# nibabel has some problem, but we might be lucky with
# pynifti below. if not, we have stored the exception
# and raise it below
img = None
pass
else:
# assume this is an image already
img = src
if isinstance(img, nibabel.spatialimages.SpatialImage):
# nibabel image, dissect and return pieces
return _get_txyz_shaped(img.get_data()), img.get_header()
def test_anova(self):
"""Do some extended testing of OneWayAnova
in particular -- compound estimation
"""
m = OneWayAnova() # default must be not compound ?
mc = CompoundOneWayAnova()
ds = datasets['uni2medium']
# For 2 labels it must be identical for both and equal to
# simple OneWayAnova
a, ac = m(ds), mc(ds)
self.failUnless(a.shape == (1, ds.nfeatures))
self.failUnless(ac.shape == (len(ds.UT), ds.nfeatures))
assert_array_equal(ac[0], ac[1])
assert_array_equal(a, ac[1])
# check for p-value attrs
if externals.exists('scipy'):
assert_true('fprob' in a.fa.keys())
assert_equal(len(ac.fa), len(ac))
ds = datasets['uni4large']
ac = mc(ds)
if cfg.getboolean('tests', 'labile', default='yes'):
# All non-bogus features must be high for a corresponding feature
self.failUnless(
(ac.samples[np.arange(4),
np.array(ds.a.nonbogus_features)] >= 1).all())
# All features should have slightly but different CompoundAnova
# values. I really doubt that there will be a case when this
# test would fail just to being 'labile'
self.failUnless(np.max(np.std(ac, axis=1)) > 0,
msg='In compound anova, we should get different'
' results for different labels. Got %s' % ac)
class PolySGKernel(_BasicSGKernel):
"""Polynomial kernel: K(a,b) = (a*b.T + c)**degree
c is 1 if and only if 'inhomogenous' is True
"""
__kernel_cls__ = sgk.PolyKernel
__kernel_name__ = 'poly'
__kp_order__ = ('degree', 'inhomogenous')
degree = Parameter(2,
allowedtype=int,
doc="Polynomial order of the kernel")
inhomogenous = Parameter(True,
allowedtype=bool,
doc="Whether +1 is added within the expression")
if not exists('sg ge 0.6.5'):
use_normalization = Parameter(False,
allowedtype=bool,
doc="Optional normalization")
__kp_order__ = __kp_order__ + ('use_normalization', )
def __init__(self, **kwargs):
# Necessary for proper docstring construction
_BasicSGKernel.__init__(self, **kwargs)
# vi: set ft=python sts=4 ts=4 sw=4 et:
### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
# See COPYING file distributed along with the PyMVPA package for the
# copyright and license terms.
#
### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Import griddata with preference to the version from matplotlib
"""
__docformat__ = 'restructuredtext'
import sys
from mvpa.base import externals
if externals.exists('griddata', raise_=True):
if __debug__:
from mvpa.base import debug
try:
if sys.version_info[:2] >= (2, 5):
# enforce absolute import
griddata = __import__('griddata', globals(), locals(), [],
0).griddata
else:
# little trick to be able to import 'griddata' package (which
# has same name)
oldname = __name__
# crazy name with close to zero possibility to cause whatever
__name__ = 'iaugf9zrkjsbdv91'
try:
from mvpa.base import externals
# XXX local rename is due but later on
from mvpa.base.collections import Collection as BaseCollection
if __debug__:
from mvpa.base import debug
# XXX
# To debug references on top level -- useful to keep around for now,
# don't remove until refactoring is complete
import sys
_debug_references = 'ATTRREFER' in debug.active
_debug_shits = [] # remember all to don't complaint twice
import traceback
_in_ipython = externals.exists('running ipython env')
# Separators around definitions, needed for ReST, but bogus for
# interactive sessions
_def_sep = ('`', '')[int(_in_ipython)]
_object_getattribute = object.__getattribute__
_object_setattr = object.__setattr__
###################################################################
# Collections
#
# TODO:
# - refactor: use base.collections and unify this to that
# - minimize interface
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
# See COPYING file distributed along with the PyMVPA package for the
# copyright and license terms.
#
### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Various utilities to help plotting"""
__docformat__ = 'restructuredtext'
from mvpa.base import externals
externals.exists("pylab", raise_=True)
import pylab as pl
interactive_backends = ['GTKAgg', 'TkAgg']
# Backends can be modified only prior importing matplotlib, so it is
# safe to just assign current backend right here
mpl_backend = pl.matplotlib.get_backend()
mpl_backend_isinteractive = mpl_backend in interactive_backends
if mpl_backend_isinteractive:
Pioff = pl.ioff
def Pion():
"""Little helper to call pl.draw() and pl.ion() if backend is interactive
"""
pl.draw()
pl.ion()
# vi: set ft=python sts=4 ts=4 sw=4 et:
### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
# See COPYING file distributed along with the PyMVPA package for the
# copyright and license terms.
#
### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Mapper for data detrending."""
__docformat__ = 'restructuredtext'
import numpy as np
from operator import isSequenceType
from mvpa.base import externals
if externals.exists('scipy', raise_=True):
# if we construct the polynomials ourselves, we wouldn't need scipy here
from scipy.special import legendre
from mvpa.base.dochelpers import _str, borrowkwargs
from mvpa.mappers.base import Mapper
class PolyDetrendMapper(Mapper):
"""Mapper for regression-based removal of polynomial trends.
Noteworthy features are the possibility for chunk-wise detrending, optional
regressors, and the ability to use positional information about the samples
from the dataset.
Any sample attribute from the to be mapped dataset can be used to define
"""Creating simple PDF reports using reportlab
"""
__docformat__ = 'restructuredtext'
import os
from datetime import datetime
import mvpa
from mvpa.base import externals, verbose
from mvpa.base.dochelpers import borrowkwargs
if __debug__:
from mvpa.base import debug
if externals.exists('reportlab', raise_=True):
import reportlab as rl
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Image
from reportlab.lib.styles import getSampleStyleSheet
from reportlab.lib.units import inch
# Actually current reportlab's Image can't deal directly with .pdf images
# Lets use png for now
if externals.versions['reportlab'] >= '1112.2':
_fig_ext_default = 'pdf'
else:
_fig_ext_default = 'png'
__all__ = ['rl', 'Report', 'escape_xml']
def figure(self, fig=None, name=None, savefig_kwargs={}, **kwargs):
"""Add a figure to the report
Parameters
----------
fig : None or str or `figure.Figure`
Figure to place into report: `str` is treated as a filename,
`Figure` stores it into a file under directory and embeds
into the report, and `None` takes the current figure
savefig_kwargs : dict
Additional keyword arguments to provide savefig with (e.g. dpi)
**kwargs
Passed to :class:`reportlab.platypus.Image` constructor
"""
if externals.exists('pylab', raise_=True):
import pylab as pl
figure = pl.matplotlib.figure
if fig is None:
fig = pl.gcf()
if isinstance(fig, figure.Figure):
# Create directory if needed
if not (os.path.exists(self._filename)
and os.path.isdir(self._filename)):
os.makedirs(self._filename)
# Figure out the name for image
self.__nfigures += 1
if name is None:
name = 'Figure#'
name = name.replace('#', str(self.__nfigures))
# Save image
fig_filename = os.path.join(self._filename,
'%s.%s' % (name, self.fig_ext))
if __debug__ and not self in debug.handlers:
debug("REP_",
"Saving figure '%s' into %s" % (fig, fig_filename))
fig.savefig(fig_filename, **savefig_kwargs)
# adjust fig to the one to be included
fig = fig_filename
if __debug__ and not self in debug.handlers:
debug("REP", "Adding figure '%s'" % fig)
im = Image(fig, **kwargs)
# If the inherent or provided width/height are too large -- shrink down
imsize = (im.drawWidth, im.drawHeight)
# Reduce the size if necessary so reportlab does not puke later on
r = [float(d) / m for d, m in zip(imsize, self.pagesize)]
maxr = max(r)
if maxr > 1.0:
if __debug__ and not self in debug.handlers:
debug("REP_", "Shrinking figure by %.3g" % maxr)
im.drawWidth /= maxr
im.drawHeight /= maxr
self._story.append(im)
