def test_null_dist_prob(self, null):
"""Testing null dist probability"""
if not isinstance(null, NullDist):
return
ds = datasets['uni2small']
null.fit(OneWayAnova(), ds)
# check reasonable output.
# p-values for non-bogus features should significantly different,
# while bogus (0) not
prob = null.p([20, 0, 0, 0, 0, np.nan])
# XXX this is labile! it also needs checking since the F-scores
# of the MCNullDists using normal distribution are apparently not
# distributed that way, hence the test often (if not always) fails.
if cfg.getboolean('tests', 'labile', default='yes'):
self.assertTrue(np.abs(prob[0]) < 0.05,
msg="Expected small p, got %g" % prob[0])
if cfg.getboolean('tests', 'labile', default='yes'):
self.assertTrue((np.abs(prob[1:]) > 0.05).all(),
msg="Bogus features should have insignificant p."
" Got %s" % (np.abs(prob[1:]),))
# has to have matching shape
if not isinstance(null, FixedNullDist):
# Fixed dist is univariate ATM so it doesn't care
# about dimensionality and gives 1 output value
self.assertRaises(ValueError, null.p, [5, 3, 4])
def test_null_dist_prob(self, null):
"""Testing null dist probability"""
if not isinstance(null, NullDist):
return
ds = datasets['uni2small']
null.fit(OneWayAnova(), ds)
# check reasonable output.
# p-values for non-bogus features should significantly different,
# while bogus (0) not
prob = null.p([20, 0, 0, 0, 0, np.nan])
# XXX this is labile! it also needs checking since the F-scores
# of the MCNullDists using normal distribution are apparently not
# distributed that way, hence the test often (if not always) fails.
if cfg.getboolean('tests', 'labile', default='yes'):
self.assertTrue(np.abs(prob[0]) < 0.05,
msg="Expected small p, got %g" % prob[0])
if cfg.getboolean('tests', 'labile', default='yes'):
self.assertTrue((np.abs(prob[1:]) > 0.05).all(),
msg="Bogus features should have insignificant p."
" Got %s" % (np.abs(prob[1:]), ))
# has to have matching shape
if not isinstance(null, FixedNullDist):
# Fixed dist is univariate ATM so it doesn't care
# about dimensionality and gives 1 output value
self.assertRaises(ValueError, null.p, [5, 3, 4])
def test_simple_som(self):
colors = np.array([[0., 0., 0.], [0., 0., 1.], [0., 1., 0.],
[1., 0., 0.], [0., 1., 1.], [1., 0., 1.],
[1., 1., 0.], [1., 1., 1.]])
distance_measures = (None, lambda x, y:(x ** 3 + y ** 3) ** (1. / 3))
for distance_measure in distance_measures:
# only small SOM for speed reasons
som = SimpleSOMMapper((10, 5), 200, learning_rate=0.05)
# no acces when nothing is there
self.assertRaises(RuntimeError, som._access_kohonen)
som.train(colors)
fmapped = som.forward(colors)
self.assertTrue(fmapped.shape == (8, 2))
# reverse mapping
rmapped = som.reverse(fmapped)
if cfg.getboolean('tests', 'labile', default='yes'):
# should approximately restore the input, but could fail
# with bad initialisation
self.assertTrue((np.round(rmapped) == colors).all())
def test_simple_som(self):
colors = np.array([[0., 0., 0.], [0., 0., 1.], [0., 1., 0.],
[1., 0., 0.], [0., 1., 1.], [1., 0., 1.],
[1., 1., 0.], [1., 1., 1.]])
distance_measures = (None, lambda x, y: (x**3 + y**3)**(1. / 3))
for distance_measure in distance_measures:
# only small SOM for speed reasons
som = SimpleSOMMapper((10, 5), 200, learning_rate=0.05)
# no acces when nothing is there
self.assertRaises(RuntimeError, som._access_kohonen)
som.train(colors)
fmapped = som.forward(colors)
self.assertTrue(fmapped.shape == (8, 2))
# reverse mapping
rmapped = som.reverse(fmapped)
if cfg.getboolean('tests', 'labile', default='yes'):
# should approximately restore the input, but could fail
# with bad initialisation
self.assertTrue((np.round(rmapped) == colors).all())
def __check_rpy():
"""Check either rpy is available and also set it for the sane execution
"""
import rpy
if not cfg.getboolean('rpy', 'interactive', default=True) \
and (rpy.get_rpy_input() is rpy.rpy_io.rpy_input):
if __debug__:
debug('EXT_',
"RPy: providing dummy callback for input to return '1'")
def input1(*args):
return "1" # which is "1: abort (with core dump, if enabled)"
rpy.set_rpy_input(input1)
def test_glmnet_r():
# not the perfect dataset with which to test, but
# it will do for now.
#data = datasets['dumb2']
# for some reason the R code fails with the dumb data
data = datasets['chirp_linear']
clf = GLMNET_R()
clf.train(data)
# prediction has to be almost perfect
# test with a correlation
pre = clf.predict(data.samples)
corerr = corr_error(pre, data.targets)
if cfg.getboolean('tests', 'labile', default='yes'):
assert_true(corerr < .2)
def teardown_module(module, verbosity=None):
"tear down test fixtures"
verbosity = _get_verbosity(verbosity)
# restore warning handlers
warning.maxcount = _sys_settings['maxcount']
if verbosity < 3:
# restore warning handlers
warning.handlers = _sys_settings['handlers']
if verbosity < 4:
# restore numpy settings
np.seterr(**_sys_settings['np_errsettings'])
if cfg.getboolean('tests', 'wtf', default='no'):
sys.stderr.write(str(wtf()))
def teardown_module(module, verbosity=None):
"tear down test fixtures"
verbosity = _get_verbosity(verbosity)
# restore warning handlers
warning.maxcount = _sys_settings['maxcount']
if verbosity < 3:
# restore warning handlers
warning.handlers = _sys_settings['handlers']
if verbosity < 4:
# restore numpy settings
np.seterr(**_sys_settings['np_errsettings'])
if cfg.getboolean('tests', 'wtf', default='no'):
sys.stderr.write(str(wtf()))
def test_enet(self):
# not the perfect dataset with which to test, but
# it will do for now.
#data = datasets['dumb2']
# for some reason the R code fails with the dumb data
data = datasets['chirp_linear']
clf = ENET()
clf.train(data)
# prediction has to be almost perfect
# test with a correlation
pre = clf.predict(data.samples)
cor = pearsonr(pre, data.targets)
if cfg.getboolean('tests', 'labile', default='yes'):
self.assertTrue(cor[0] > .8)
def test_enet(self):
# not the perfect dataset with which to test, but
# it will do for now.
#data = datasets['dumb2']
# for some reason the R code fails with the dumb data
data = datasets['chirp_linear']
clf = ENET()
clf.train(data)
# prediction has to be almost perfect
# test with a correlation
pre = clf.predict(data.samples)
cor = pearsonr(pre, data.targets)
if cfg.getboolean('tests', 'labile', default='yes'):
self.assertTrue(cor[0] > .8)
def test_glmnet_r():
# not the perfect dataset with which to test, but
# it will do for now.
#data = datasets['dumb2']
# for some reason the R code fails with the dumb data
data = datasets['chirp_linear']
clf = GLMNET_R()
clf.train(data)
# prediction has to be almost perfect
# test with a correlation
pre = clf.predict(data.samples)
corerr = corr_error(pre, data.targets)
if cfg.getboolean('tests', 'labile', default='yes'):
assert_true(corerr < .2)
def __check_rpy():
"""Check either rpy is available and also set it for the sane execution
"""
#import rpy_options
#rpy_options.set_options(VERBOSE=False, SETUP_READ_CONSOLE=False) # SETUP_WRITE_CONSOLE=False)
#rpy_options.set_options(VERBOSE=False, SETUP_WRITE_CONSOLE=False) # SETUP_WRITE_CONSOLE=False)
# if not cfg.get('rpy', 'read_console', default=False):
# print "no read"
# rpy_options.set_options(SETUP_READ_CONSOLE=False)
# if not cfg.get('rpy', 'write_console', default=False):
# print "no write"
# rpy_options.set_options(SETUP_WRITE_CONSOLE=False)
import rpy
if not cfg.getboolean('rpy', 'interactive', default=True) \
and (rpy.get_rpy_input() is rpy.rpy_io.rpy_input):
if __debug__:
debug('EXT_', "RPy: providing dummy callback for input to return '1'")
def input1(*args): return "1" # which is "1: abort (with core dump, if enabled)"
rpy.set_rpy_input(input1)
def check_all_dependencies(force=False, verbosity=1):
"""
Test for all known dependencies.
Parameters
----------
force : boolean
Whether to force the test even if it has already been
performed.
"""
# loop over all known dependencies
for dep in _KNOWN:
if not exists(dep, force):
if verbosity:
warning("%s is not available." % dep)
if __debug__:
debug('EXT', 'The following optional externals are present: %s'
% [k[5:] for k in cfg.options('externals')
if k.startswith('have')
and cfg.getboolean('externals', k)])
def test_anova(self):
"""Do some extended testing of OneWayAnova
in particular -- compound estimation
"""
m = OneWayAnova() # default must be not compound ?
mc = CompoundOneWayAnova()
ds = datasets['uni2medium']
# For 2 labels it must be identical for both and equal to
# simple OneWayAnova
a, ac = m(ds), mc(ds)
self.assertTrue(a.shape == (1, ds.nfeatures))
self.assertTrue(ac.shape == (len(ds.UT), ds.nfeatures))
assert_array_equal(ac[0], ac[1])
assert_array_equal(a, ac[1])
# check for p-value attrs
if externals.exists('scipy'):
assert_true('fprob' in a.fa.keys())
assert_equal(len(ac.fa), len(ac))
ds = datasets['uni4large']
ac = mc(ds)
if cfg.getboolean('tests', 'labile', default='yes'):
# All non-bogus features must be high for a corresponding feature
self.assertTrue(
(ac.samples[np.arange(4),
np.array(ds.a.nonbogus_features)] >= 1).all())
# All features should have slightly but different CompoundAnova
# values. I really doubt that there will be a case when this
# test would fail just to being 'labile'
self.assertTrue(np.max(np.std(ac, axis=1)) > 0,
msg='In compound anova, we should get different'
' results for different labels. Got %s' % ac)
def test_anova(self):
"""Do some extended testing of OneWayAnova
in particular -- compound estimation
"""
m = OneWayAnova() # default must be not compound ?
mc = CompoundOneWayAnova()
ds = datasets['uni2medium']
# For 2 labels it must be identical for both and equal to
# simple OneWayAnova
a, ac = m(ds), mc(ds)
self.assertTrue(a.shape == (1, ds.nfeatures))
self.assertTrue(ac.shape == (len(ds.UT), ds.nfeatures))
assert_array_equal(ac[0], ac[1])
assert_array_equal(a, ac[1])
# check for p-value attrs
if externals.exists('scipy'):
assert_true('fprob' in a.fa.keys())
assert_equal(len(ac.fa), len(ac))
ds = datasets['uni4large']
ac = mc(ds)
if cfg.getboolean('tests', 'labile', default='yes'):
# All non-bogus features must be high for a corresponding feature
self.assertTrue((ac.samples[np.arange(4),
np.array(ds.a.nonbogus_features)] >= 1
).all())
# All features should have slightly but different CompoundAnova
# values. I really doubt that there will be a case when this
# test would fail just to being 'labile'
self.assertTrue(np.max(np.std(ac, axis=1)) > 0,
msg='In compound anova, we should get different'
' results for different labels. Got %s' % ac)
Dimensionality of target space
data : array, optional
Some data (should have rank high enough) to derive
rotation
"""
if nt is None:
nt = ns
# figure out some "random" rotation
d = max(ns, nt)
if data is None:
data = np.random.normal(size=(d*10, d))
_u, _s, _vh = np.linalg.svd(data[:, :d])
R = _vh[:ns, :nt]
if ns == nt:
# Test if it is indeed a rotation matrix ;)
# Lets flip first axis if necessary
if np.linalg.det(R) < 0:
R[:, 0] *= -1.0
return R
datasets = generate_testing_datasets(specs)
if cfg.getboolean('tests', 'use hdf datasets', False):
if not externals.exists('h5py'):
raise RuntimeError(
"Cannot perform HDF5 dump of all datasets in the warehouse, "
"because 'h5py' is not available")
datasets = saveload_warehouse()
print "Replaced all dataset warehouse for HDF5 loaded alternative."
error_str = ". Caught exception was: " + str(e)
else:
raise
finally:
# And restore warnings
np.seterr(**old_handling)
if __debug__:
debug('EXT', "Presence of %s is%s verified%s" %
(dep, {True:'', False:' NOT'}[result], error_str))
if not result:
if raise_:
raise exception("Required external '%s' was not found" % dep)
if issueWarning is not None \
and cfg.getboolean('externals', 'issue warning', True):
if issueWarning is True:
warning("Required external '%s' was not found" % dep)
else:
warning(issueWarning)
# store result in config manager
if not cfg.has_section('externals'):
cfg.add_section('externals')
if result:
cfg.set('externals', 'have ' + dep, 'yes')
else:
cfg.set('externals', 'have ' + dep, 'no')
return result
# store the whole datasets warehouse in one hdf5 file
hdf = h5py.File(pathjoin(tempdir, 'myhdf5.hdf5'), 'w')
for d in datasets:
obj2hdf(hdf, datasets[d], d)
hdf.close()
hdf = h5py.File(pathjoin(tempdir, 'myhdf5.hdf5'), 'r')
rc_ds = {}
for d in hdf:
rc_ds[d] = hdf2obj(hdf[d])
hdf.close()
#cleanup temp dir
shutil.rmtree(tempdir, ignore_errors=True)
# return the reconstructed datasets (for use in datasets warehouse)
return rc_ds
datasets = generate_testing_datasets(specs)
if cfg.getboolean('tests', 'use hdf datasets', False):
if not externals.exists('h5py'):
raise RuntimeError(
"Cannot perform HDF5 dump of all datasets in the warehouse, "
"because 'h5py' is not available")
datasets = saveload_warehouse()
print "Replaced all dataset warehouse for HDF5 loaded alternative."
def collect_nose_tests(verbosity=1):
"""Return list of tests which are pure nose-based
"""
tests = [
# Basic data structures/manipulators
'test_base',
'test_collections',
'test_attrmap',
# Datasets
'test_datasetng',
'test_datasetfx',
'test_dataset_formats',
'test_splitter',
'test_generators',
'test_niftidataset',
'test_eepdataset',
'test_erdataset',
'test_datasrcs',
# Classifiers
'test_multiclf',
# Misc supporting
'test_neighborhood',
'test_stats',
'test_stats_sp',
# Mappers
'test_mapper',
'test_mapper_sp',
'test_arraymapper',
'test_boxcarmapper',
'test_prototypemapper',
'test_fxmapper',
'test_zscoremapper',
'test_waveletmapper',
'test_mdp',
'test_filters',
'test_staticprojection',
# Learners
'test_enet',
'test_spam',
'test_lars',
'test_glmnet',
'test_kernel',
'test_svmkernels',
'test_senses',
# Algorithms
'test_emp_null',
'test_clfcrossval',
# IO
'test_iohelpers',
'test_hdf5',
'test_hdf5_clf',
# Measures
'test_transerror',
'test_datameasure',
'test_dcov',
# Misc
'test_misc',
'test_errorfx',
'test_testing',
'test_usecases',
'test_surfing',
'test_surfing_afni',
'test_surfing_voxelselection'
]
if not cfg.getboolean('tests', 'lowmem', default='no'):
tests += ['test_atlases']
return tests
def enhanced_doc_string(item, *args, **kwargs):
"""Generate enhanced doc strings for various items.
Parameters
----------
item : str or class
What object requires enhancing of documentation
*args : list
Includes base classes to look for parameters, as well, first item
must be a dictionary of locals if item is given by a string
force_extend : bool
Either to force looking for the documentation in the parents.
By default force_extend = False, and lookup happens only if kwargs
is one of the arguments to the respective function (e.g. item.__init__)
skip_params : list of str
List of parameters (in addition to [kwargs]) which should not
be added to the documentation of the class.
It is to be used from a collector, ie whenever class is already created
"""
# Handling of arguments
if len(kwargs):
if set(kwargs.keys()).issubset(set(['force_extend'])):
raise ValueError, "Got unknown keyword arguments (smth among %s)" \
" in enhanced_doc_string." % kwargs
force_extend = kwargs.get('force_extend', False)
skip_params = kwargs.get('skip_params', [])
# XXX make it work also not only with classes but with methods as well
if isinstance(item, basestring):
if len(args)<1 or not isinstance(args[0], dict):
raise ValueError, \
"Please provide locals for enhanced_doc_string of %s" % item
name = item
lcl = args[0]
args = args[1:]
elif hasattr(item, "im_class"):
# bound method
raise NotImplementedError, \
"enhanced_doc_string is not yet implemented for methods"
elif hasattr(item, "__name__"):
name = item.__name__
lcl = item.__dict__
else:
raise ValueError, "Don't know how to extend docstring for %s" % item
# check whether docstring magic is requested or not
if not cfg.getboolean('doc', 'pimp docstrings', True):
return lcl['__doc__']
if __debug__:
debug('DOCH', 'Processing docstrings of %s' % name)
#return lcl['__doc__']
rst_lvlmarkup = ["=", "-", "_"]
# would then be called for any child... ok - ad hoc for SVM???
if hasattr(item, '_customize_doc') and name=='SVM':
item._customize_doc()
initdoc = ""
if lcl.has_key('__init__'):
func = lcl['__init__']
initdoc = func.__doc__
skip_params += lcl.get('__init__doc__exclude__', [])
# either to extend arguments
# do only if kwargs is one of the arguments
# in python 2.5 args are no longer in co_names but in varnames
extend_args = force_extend or \
'kwargs' in (func.func_code.co_names +
func.func_code.co_varnames)
if __debug__ and not extend_args:
debug('DOCH',
'Not extending parameters for __init__ of %s',
(name,))
if initdoc is None:
initdoc = "Initialize instance of %s" % name
initdoc, params, suffix = _split_out_parameters(initdoc)
params_list = _parse_parameters(params)
known_params = set([i[0] for i in params_list])
# If there are additional ones:
if lcl.has_key('_paramsdoc'):
params_list += [i for i in lcl['_paramsdoc']
if not (i[0] in known_params)]
known_params = set([i[0] for i in params_list])
# no need for placeholders
skip_params = set(skip_params + ['kwargs', '**kwargs'])
# XXX we do evil check here, refactor code to separate
# regressions out of the classifiers, and making
# retrainable flag not available for those classes which
# can't actually do retraining. Although it is not
# actually that obvious for Meta Classifiers
if hasattr(item, '__tags__'):
clf_internals = item.__tags__
skip_params.update([i for i in ('retrainable',)
if not (i in clf_internals)])
known_params.update(skip_params)
if extend_args:
# go through all the parents and obtain their init parameters
parent_params_list = []
for i in args:
if hasattr(i, '__init__'):
# XXX just assign within a class to don't redo without need
initdoc_ = i.__init__.__doc__
if initdoc_ is None:
continue
splits_ = _split_out_parameters(initdoc_)
params_ = splits_[1]
parent_params_list += _parse_parameters(params_.lstrip())
# extend with ones which are not known to current init
for i, v in parent_params_list:
if not (i in known_params):
params_list += [(i, v)]
known_params.update([i])
# if there are parameters -- populate the list
if len(params_list):
params_ = '\n'.join([i[1].rstrip() for i in params_list
if not i[0] in skip_params])
initdoc += "\n\n%s\n" \
% _rst_section('Parameters') + _indent(params_)
if suffix != "":
initdoc += "\n\n" + suffix
initdoc = handle_docstring(initdoc)
# Finally assign generated doc to the constructor
lcl['__init__'].__doc__ = initdoc
docs = [ handle_docstring(lcl['__doc__']) ]
# Optionally populate the class documentation with it
if __add_init2doc and initdoc != "":
docs += [ _rst_underline('Constructor information for `%s` class'
% name, rst_lvlmarkup[2]),
initdoc ]
# Add information about the ca if available
if lcl.has_key('_cadoc') and len(item._cadoc):
# to don't conflict with Notes section if such was already
# present
lcldoc = lcl['__doc__'] or ''
if not 'Notes' in lcldoc:
section_name = _rst_section('Notes')
else:
section_name = '\n' # just an additional newline
# no indent is necessary since ca list must be already indented
docs += ['%s\nAvailable conditional attributes:' % section_name,
handle_docstring(item._cadoc)]
# Deprecated -- but actually we might like to have it in ipython
# mode may be?
if False: #len(args):
bc_intro = _rst(' ') + 'Please refer to the documentation of the ' \
'base %s for more information:' \
% (single_or_plural('class', 'classes', len(args)))
docs += ['\n' + _rst_section('See Also'),
bc_intro,
' ' + ',\n '.join(['%s%s.%s%s%s' % (_rst(':class:`~'),
i.__module__,
i.__name__,
_rst('`'),
_rst_sep)
for i in args])
]
itemdoc = '\n\n'.join(docs)
# remove some bogus new lines -- never 3 empty lines in doc are useful
result = re.sub("\s*\n\s*\n\s*\n", "\n\n", itemdoc)
return result
def do_sweep(*args_, **kwargs_):
"""Perform sweeping over provided keyword arguments
"""
def untrain_clf(argvalue):
"""Little helper"""
if isinstance(argvalue, Classifier):
# clear classifier after its use -- just to be sure ;-)
argvalue.params.retrainable = False
argvalue.untrain()
failed_tests = {}
skipped_tests = []
report_progress = cfg.get('tests', 'verbosity', default=1) > 1
for argname in list(kwargs.keys()):
for argvalue in kwargs[argname]:
if isinstance(argvalue, Classifier):
# clear classifier before its use
argvalue.untrain()
if isinstance(argvalue, ClassWithCollections):
argvalue.ca.reset()
# update kwargs_
kwargs_[argname] = argvalue
# do actual call
try:
if __debug__:
debug(
'TEST', 'Running %s on args=%r and kwargs=%r' %
(method.__name__, args_, kwargs_))
method(*args_, **kwargs_)
status = '+'
except SkipTest as e:
skipped_tests += [e]
status = 'S'
except AssertionError as e:
status = 'F'
estr = str(e)
etype, value, tb = sys.exc_info()
# literal representation of exception tb, so
# we could group them later on
eidstr = ' '.join([
l for l in tbm.format_exception(etype, value, tb)
if not ('do_sweep' in l or 'unittest.py' in l
or 'AssertionError' in l
or 'Traceback (most' in l)
])
# Store exception information for later on groupping
if not eidstr in failed_tests:
failed_tests[eidstr] = []
sargvalue = safe_str(argvalue)
if not (__debug__ and 'TEST' in debug.active):
# by default lets make it of sane length
if len(sargvalue) > 100:
sargvalue = sargvalue[:95] + ' ...'
failed_tests[eidstr].append(
# skip top-most tb in sweep_args
(argname, sargvalue, tb.tb_next, estr))
if __debug__:
msg = "%s on %s=%s" % (estr, argname,
safe_str(argvalue))
debug('TEST',
'Failed unittest: %s\n%s' % (eidstr, msg))
if report_progress:
sys.stdout.write(status)
sys.stdout.flush()
untrain_clf(argvalue)
# TODO: handle different levels of unittests properly
if cfg.getboolean('tests', 'quick', False):
# on TESTQUICK just run test for 1st entry in the list,
# the rest are omitted
# TODO: proper partitioning of unittests
break
if report_progress:
sys.stdout.write(' ')
sys.stdout.flush()
if len(failed_tests):
# Lets now create a single AssertionError exception
# which would nicely incorporate all failed exceptions
multiple = len(failed_tests) != 1 # is it unique?
# if so, we don't need to reinclude traceback since it
# would be spitted out anyways below
estr = ""
cestr = "lead to failures of unittest %s" % method.__name__
if multiple:
estr += "\n Different scenarios %s "\
"(specific tracebacks are below):" % cestr
else:
estr += "\n Single scenario %s:" % cestr
for ek, els in failed_tests.items():
estr += '\n'
if multiple:
estr += ek
estr += " on\n %s" % (
" ".join([
"%s=%s%s\n" % (
ea,
eav,
# Why didn't I just do regular for loop? ;)
":\n ".join(
[xx for xx in [' ', es] if xx != '']))
for ea, eav, etb, es in els
]))
# take first one... they all should be identical
etb = els[0][2]
raise AssertionError(estr).with_traceback(etb)
if len(skipped_tests):
# so if nothing has failed, lets at least report that some were
# skipped -- for now just a simple SkipTest message
raise SkipTest("%d tests were skipped in testing %s" %
(len(skipped_tests), method.__name__))
# plot the histogram
plot_bars(H.T, xloc=bin_left, width=bin_width, yerr='std')
# show the Gaussians
x = np.linspace(0, 1, 100)
# first gaussian
pl.plot(x, params[0] * norm.pdf(x, params[1], params[2]), "r-", zorder=2)
pl.axvline(params[1], color='r', linestyle='--', alpha=0.6)
# second gaussian
pl.plot(x, params[3] * norm.pdf(x, params[4], params[5]), "b-", zorder=3)
pl.axvline(params[4], color='b', linestyle='--', alpha=0.6)
# dual gaussian
pl.plot(x, dual_gaussian(x, *params), "k--", alpha=0.5, zorder=1)
pl.xlim(0, 1)
pl.ylim(ymin=0)
pl.title('Dual Gaussian fit of searchlight accuracies')
if cfg.getboolean('examples', 'interactive', True):
# show the cool figures
pl.show()
"""
And this is how it looks like.
.. image:: ../pics/ex_curvefitting_searchlight.*
:align: center
:alt: Dual Gaussian fit of searchlight accuracies
"""
def do_sweep(*args_, **kwargs_):
"""Perform sweeping over provided keyword arguments
"""
def untrain_clf(argvalue):
"""Little helper"""
if isinstance(argvalue, Classifier):
# clear classifier after its use -- just to be sure ;-)
argvalue.params.retrainable = False
argvalue.untrain()
failed_tests = {}
skipped_tests = []
for argname in kwargs.keys():
for argvalue in kwargs[argname]:
if isinstance(argvalue, Classifier):
# clear classifier before its use
argvalue.untrain()
if isinstance(argvalue, ClassWithCollections):
argvalue.ca.reset()
# update kwargs_
kwargs_[argname] = argvalue
# do actual call
try:
if __debug__:
debug('TEST', 'Running %s on args=%r and kwargs=%r'
% (method.__name__, args_, kwargs_))
method(*args_, **kwargs_)
except SkipTest, e:
skipped_tests += [e]
except AssertionError, e:
estr = str(e)
etype, value, tb = sys.exc_info()
# literal representation of exception tb, so
# we could group them later on
eidstr = ' '.join(
[l for l in tbm.format_exception(etype, value, tb)
if not ('do_sweep' in l
or 'unittest.py' in l
or 'AssertionError' in l
or 'Traceback (most' in l)])
# Store exception information for later on groupping
if not eidstr in failed_tests:
failed_tests[eidstr] = []
sargvalue = str(argvalue)
if not (__debug__ and 'TEST' in debug.active):
# by default lets make it of sane length
if len(sargvalue) > 100:
sargvalue = sargvalue[:95] + ' ...'
failed_tests[eidstr].append(
# skip top-most tb in sweep_args
(argname, sargvalue, tb.tb_next, estr))
if __debug__:
msg = "%s on %s=%s" % (estr, argname, argvalue)
debug('TEST', 'Failed unittest: %s\n%s'
% (eidstr, msg))
untrain_clf(argvalue)
# TODO: handle different levels of unittests properly
if cfg.getboolean('tests', 'quick', False):
# on TESTQUICK just run test for 1st entry in the list,
# the rest are omitted
# TODO: proper partitioning of unittests
break
def collect_nose_tests(verbosity=1):
"""Return tests which are purely nose-based (now it is actually a mix)
"""
tests = [
# Basic data structures/manipulators
'test_base',
'test_collections',
'test_attrmap',
'test_constraints',
# Datasets
'test_bids',
'test_giftidataset',
'test_datasetng',
'test_datasetfx',
'test_dataset_formats',
'test_splitter',
'test_generators',
'test_niftidataset',
'test_cosmo',
'test_eepdataset',
'test_erdataset',
'test_datasrcs',
# Classifiers
'test_multiclf',
'test_smlr',
# Misc supporting
'test_neighborhood',
'test_stats',
'test_stats_sp',
# Mappers
'test_mapper',
'test_mapper_sp',
'test_arraymapper',
'test_boxcarmapper',
'test_prototypemapper',
'test_fxmapper',
'test_zscoremapper',
'test_waveletmapper',
'test_mdp',
'test_filters',
'test_staticprojection',
# Learners
'test_compound',
'test_enet',
'test_glmmapper',
'test_lars',
'test_glmnet',
'test_kernel',
'test_svmkernels',
'test_senses',
# Algorithms
'test_emp_null',
'test_clfcrossval',
'test_group_clusterthr',
# IO
'test_iohelpers',
'test_hdf5',
'test_hdf5_clf',
'test_openfmri',
# Measures
'test_transerror',
'test_datameasure',
'test_dcov',
'test_corrstability',
'test_fxmeasure',
'test_rsa',
# Misc
'test_cmdline_ttest',
'test_lib_afni',
'test_misc_scatter',
'test_misc',
'test_errorfx',
'test_testing',
'test_usecases',
'test_surfing',
'test_surfing_afni',
'test_surfing_voxelselection',
'test_surfing_surface',
'test_eeglab',
'test_progress',
'test_winner',
'test_viz',
]
"""Return list of tests which are pure nose-based
"""
if not cfg.getboolean('tests', 'lowmem', default='no'):
tests += ['test_atlases']
return tests
def collect_nose_tests(verbosity=1):
"""Return list of tests which are pure nose-based
"""
tests = [
# Basic data structures/manipulators
'test_base',
'test_collections',
'test_attrmap',
# Datasets
'test_datasetng',
'test_datasetfx',
'test_dataset_formats',
'test_splitter',
'test_generators',
'test_niftidataset',
'test_eepdataset',
'test_erdataset',
'test_datasrcs',
# Classifiers
'test_multiclf',
# Misc supporting
'test_neighborhood',
'test_stats',
'test_stats_sp',
# Mappers
'test_mapper',
'test_mapper_sp',
'test_arraymapper',
'test_boxcarmapper',
'test_prototypemapper',
'test_fxmapper',
'test_zscoremapper',
'test_waveletmapper',
'test_mdp',
'test_filters',
'test_staticprojection',
# Learners
'test_enet',
'test_spam',
'test_lars',
'test_glmnet',
'test_kernel',
'test_svmkernels',
'test_senses',
# Algorithms
'test_emp_null',
'test_clfcrossval',
# IO
'test_iohelpers',
'test_hdf5',
'test_hdf5_clf',
# Measures
'test_transerror',
'test_datameasure',
'test_dcov',
# Misc
'test_misc',
'test_errorfx',
'test_testing',
'test_usecases',
'test_surfing',
'test_surfing_afni',
'test_surfing_voxelselection',
'test_eeglab'
]
if not cfg.getboolean('tests', 'lowmem', default='no'):
tests += ['test_atlases']
return tests
def enhanced_doc_string(item, *args, **kwargs):
"""Generate enhanced doc strings for various items.
Parameters
----------
item : str or class
What object requires enhancing of documentation
*args : list
Includes base classes to look for parameters, as well, first item
must be a dictionary of locals if item is given by a string
force_extend : bool
Either to force looking for the documentation in the parents.
By default force_extend = False, and lookup happens only if kwargs
is one of the arguments to the respective function (e.g. item.__init__)
skip_params : list of str
List of parameters (in addition to [kwargs]) which should not
be added to the documentation of the class.
It is to be used from a collector, ie whenever class is already created
"""
# Handling of arguments
if len(kwargs):
if set(kwargs.keys()).issubset(set(['force_extend'])):
raise ValueError, "Got unknown keyword arguments (smth among %s)" \
" in enhanced_doc_string." % kwargs
force_extend = kwargs.get('force_extend', False)
skip_params = kwargs.get('skip_params', [])
# XXX make it work also not only with classes but with methods as well
if isinstance(item, basestring):
if len(args)<1 or not isinstance(args[0], dict):
raise ValueError, \
"Please provide locals for enhanced_doc_string of %s" % item
name = item
lcl = args[0]
args = args[1:]
elif hasattr(item, "im_class"):
# bound method
raise NotImplementedError, \
"enhanced_doc_string is not yet implemented for methods"
elif hasattr(item, "__name__"):
name = item.__name__
lcl = item.__dict__
else:
raise ValueError, "Don't know how to extend docstring for %s" % item
# check whether docstring magic is requested or not
if not cfg.getboolean('doc', 'pimp docstrings', True):
return lcl['__doc__']
if __debug__:
debug('DOCH', 'Processing docstrings of %s' % name)
#return lcl['__doc__']
rst_lvlmarkup = ["=", "-", "_"]
# would then be called for any child... ok - ad hoc for SVM???
if hasattr(item, '_customize_doc') and name=='SVM':
item._customize_doc()
initdoc = ""
if lcl.has_key('__init__'):
func = lcl['__init__']
initdoc = func.__doc__
skip_params += lcl.get('__init__doc__exclude__', [])
# either to extend arguments
# do only if kwargs is one of the arguments
# in python 2.5 args are no longer in co_names but in varnames
extend_args = force_extend or \
'kwargs' in (func.func_code.co_names +
func.func_code.co_varnames)
if __debug__ and not extend_args:
debug('DOCH',
'Not extending parameters for __init__ of %s',
(name,))
if initdoc is None:
initdoc = "Initialize instance of %s" % name
initdoc, params, suffix = _split_out_parameters(initdoc)
params_list = _parse_parameters(params)
known_params = set([i[0] for i in params_list])
# If there are additional ones:
if lcl.has_key('_paramsdoc'):
params_list += [i for i in lcl['_paramsdoc']
if not (i[0] in known_params)]
known_params = set([i[0] for i in params_list])
# no need for placeholders
skip_params = set(skip_params + ['kwargs', '**kwargs'])
# XXX we do evil check here, refactor code to separate
# regressions out of the classifiers, and making
# retrainable flag not available for those classes which
# can't actually do retraining. Although it is not
# actually that obvious for Meta Classifiers
if hasattr(item, '__tags__'):
clf_internals = item.__tags__
skip_params.update([i for i in ('retrainable',)
if not (i in clf_internals)])
known_params.update(skip_params)
if extend_args:
# go through all the parents and obtain their init parameters
parent_params_list = []
for i in args:
if hasattr(i, '__init__'):
# XXX just assign within a class to don't redo without need
initdoc_ = i.__init__.__doc__
if initdoc_ is None:
continue
splits_ = _split_out_parameters(initdoc_)
params_ = splits_[1]
parent_params_list += _parse_parameters(params_.lstrip())
# extend with ones which are not known to current init
for i, v in parent_params_list:
if not (i in known_params):
params_list += [(i, v)]
known_params.update([i])
# if there are parameters -- populate the list
if len(params_list):
params_ = '\n'.join([i[1].rstrip() for i in params_list
if not i[0] in skip_params])
initdoc += "\n\n%s\n" \
% _rst_section('Parameters') + _indent(params_)
if suffix != "":
initdoc += "\n\n" + suffix
initdoc = handle_docstring(initdoc)
# Finally assign generated doc to the constructor
lcl['__init__'].__doc__ = initdoc
docs = [ handle_docstring(lcl['__doc__']) ]
# Optionally populate the class documentation with it
if __add_init2doc and initdoc != "":
docs += [ _rst_underline('Constructor information for `%s` class'
% name, rst_lvlmarkup[2]),
initdoc ]
# Add information about the ca if available
if lcl.has_key('_cadoc') and len(item._cadoc):
# to don't conflict with Notes section if such was already
# present
lcldoc = lcl['__doc__'] or ''
if not 'Notes' in lcldoc:
section_name = _rst_section('Notes')
else:
section_name = '\n' # just an additional newline
# no indent is necessary since ca list must be already indented
docs += ['%s\nAvailable conditional attributes:' % section_name,
handle_docstring(item._cadoc)]
# Deprecated -- but actually we might like to have it in ipython
# mode may be?
if False: #len(args):
bc_intro = _rst(' ') + 'Please refer to the documentation of the ' \
'base %s for more information:' \
% (single_or_plural('class', 'classes', len(args)))
docs += ['\n' + _rst_section('See Also'),
bc_intro,
' ' + ',\n '.join(['%s%s.%s%s%s' % (_rst(':class:`~'),
i.__module__,
i.__name__,
_rst('`'),
_rst_sep)
for i in args])
]
itemdoc = '\n\n'.join(docs)
# remove some bogus new lines -- never 3 empty lines in doc are useful
result = re.sub("\s*\n\s*\n\s*\n", "\n\n", itemdoc)
return result
def collect_nose_tests(verbosity=1):
"""Return tests which are purely nose-based (now it is actually a mix)
"""
tests = [
# Basic data structures/manipulators
'test_base',
'test_collections',
'test_attrmap',
'test_constraints',
# Datasets
'test_bids',
'test_giftidataset',
'test_datasetng',
'test_datasetfx',
'test_dataset_formats',
'test_splitter',
'test_generators',
'test_niftidataset',
'test_cosmo',
'test_eepdataset',
'test_erdataset',
'test_datasrcs',
# Classifiers
'test_multiclf',
'test_smlr',
# Misc supporting
'test_neighborhood',
'test_stats',
'test_stats_sp',
# Mappers
'test_mapper',
'test_mapper_sp',
'test_arraymapper',
'test_boxcarmapper',
'test_prototypemapper',
'test_fxmapper',
'test_zscoremapper',
'test_waveletmapper',
'test_mdp',
'test_filters',
'test_staticprojection',
# Learners
'test_compound',
'test_enet',
'test_glmmapper',
'test_lars',
'test_glmnet',
'test_kernel',
'test_svmkernels',
'test_senses',
# Algorithms
'test_emp_null',
'test_clfcrossval',
'test_group_clusterthr',
# IO
'test_iohelpers',
'test_hdf5',
'test_hdf5_clf',
'test_openfmri',
# Measures
'test_transerror',
'test_datameasure',
'test_dcov',
'test_corrstability',
'test_fxmeasure',
'test_rsa',
# Misc
'test_cmdline_ttest',
'test_lib_afni',
'test_misc_scatter',
'test_misc_plot',
'test_misc',
'test_errorfx',
'test_testing',
'test_usecases',
'test_surfing',
'test_surfing_afni',
'test_surfing_voxelselection',
'test_surfing_surface',
'test_eeglab',
'test_progress',
'test_winner',
'test_viz',
]
"""Return list of tests which are pure nose-based
"""
if not cfg.getboolean('tests', 'lowmem', default='no'):
tests += ['test_atlases']
return tests
def exists(dep, force=False, raise_=False, issueWarning=None,
exception=RuntimeError):
"""
Test whether a known dependency is installed on the system.
This method allows us to test for individual dependencies without
testing all known dependencies. It also ensures that we only test
for a dependency once.
Parameters
----------
dep : string or list of string
The dependency key(s) to test.
force : boolean
Whether to force the test even if it has already been
performed.
raise_ : boolean, str
Whether to raise an exception if dependency is missing.
If True, it is still conditioned on the global setting
MVPA_EXTERNALS_RAISE_EXCEPTION, while would raise exception
if missing despite the configuration if 'always'.
issueWarning : string or None or True
If string, warning with given message would be thrown.
If True, standard message would be used for the warning
text.
exception : exception, optional
What exception to raise. Defaults to RuntimeError
"""
# if we are provided with a list of deps - go through all of them
if isinstance(dep, (list, tuple)):
results = [ exists(dep_, force, raise_) for dep_ in dep ]
return bool(reduce(lambda x, y: x and y, results, True))
# where to look in cfg
cfgid = 'have ' + dep
# pre-handle raise_ according to the global settings and local argument
if isinstance(raise_, str):
if raise_.lower() == 'always':
raise_ = True
else:
raise ValueError("Unknown value of raise_=%s. "
"Must be bool or 'always'" % raise_)
else: # must be bool conditioned on the global settings
raise_ = (raise_
and cfg.getboolean('externals', 'raise exception', True))
# prevent unnecessary testing
if cfg.has_option('externals', cfgid) \
and not cfg.getboolean('externals', 'retest', default='no') \
and not force:
if __debug__:
debug('EXT', "Skip retesting for '%s'." % dep)
# check whether an exception should be raised, even though the external
# was already tested previously
if not cfg.getboolean('externals', cfgid) and raise_:
raise exception("Required external '%s' was not found" % dep)
return cfg.getboolean('externals', cfgid)
# determine availability of external (non-cached)
# default to 'not found'
result = False
if dep not in _KNOWN:
raise ValueError("%r is not a known dependency key." % (dep,))
else:
# try and load the specific dependency
if __debug__:
debug('EXT', "Checking for the presence of %s" % dep)
# Exceptions which are silently caught while running tests for externals
_caught_exceptions = [ImportError, AttributeError, RuntimeError]
try:
# Suppress NumPy warnings while testing for externals
old_handling = np.seterr(all="ignore")
error_str = ''
try:
exec _KNOWN[dep]
result = True
except tuple(_caught_exceptions), e:
error_str = ". Caught exception was: " + str(e)
except Exception, e:
# Add known ones by their names so we don't need to
# actually import anything manually to get those classes
if e.__class__.__name__ in ['RPy_Exception', 'RRuntimeError',
'RPy_RException']:
_caught_exceptions += [e.__class__]
error_str = ". Caught exception was: " + str(e)
else:
raise
# plot the histogram
plot_bars(H.T, xloc=bin_left, width=bin_width, yerr='std')
# show the Gaussians
x = np.linspace(0, 1, 100)
# first gaussian
pl.plot(x, params[0] * norm.pdf(x, params[1], params[2]), "r-", zorder=2)
pl.axvline(params[1], color='r', linestyle='--', alpha=0.6)
# second gaussian
pl.plot(x, params[3] * norm.pdf(x, params[4], params[5]), "b-", zorder=3)
pl.axvline(params[4], color='b', linestyle='--', alpha=0.6)
# dual gaussian
pl.plot(x, dual_gaussian(x, *params), "k--", alpha=0.5, zorder=1)
pl.xlim(0, 1)
pl.ylim(ymin=0)
pl.title('Dual Gaussian fit of searchlight accuracies')
if cfg.getboolean('examples', 'interactive', True):
# show the cool figures
pl.show()
"""
And this is how it looks like.
.. image:: ../pics/ex_curvefitting_searchlight.*
:align: center
:alt: Dual Gaussian fit of searchlight accuracies
"""
def do_sweep(*args_, **kwargs_):
"""Perform sweeping over provided keyword arguments
"""
def untrain_clf(argvalue):
"""Little helper"""
if isinstance(argvalue, Classifier):
# clear classifier after its use -- just to be sure ;-)
argvalue.params.retrainable = False
argvalue.untrain()
failed_tests = {}
skipped_tests = []
report_progress = cfg.get('tests', 'verbosity', default=1) > 1
for argname in kwargs.keys():
for argvalue in kwargs[argname]:
if isinstance(argvalue, Classifier):
# clear classifier before its use
argvalue.untrain()
if isinstance(argvalue, ClassWithCollections):
argvalue.ca.reset()
# update kwargs_
kwargs_[argname] = argvalue
# do actual call
try:
if __debug__:
debug(
'TEST', 'Running %s on args=%r and kwargs=%r' %
(method.__name__, args_, kwargs_))
method(*args_, **kwargs_)
status = '+'
except SkipTest, e:
skipped_tests += [e]
status = 'S'
except AssertionError, e:
status = 'F'
estr = str(e)
etype, value, tb = sys.exc_info()
# literal representation of exception tb, so
# we could group them later on
eidstr = ' '.join([
l for l in tbm.format_exception(etype, value, tb)
if not ('do_sweep' in l or 'unittest.py' in l
or 'AssertionError' in l
or 'Traceback (most' in l)
])
# Store exception information for later on groupping
if not eidstr in failed_tests:
failed_tests[eidstr] = []
sargvalue = str(argvalue)
if not (__debug__ and 'TEST' in debug.active):
# by default lets make it of sane length
if len(sargvalue) > 100:
sargvalue = sargvalue[:95] + ' ...'
failed_tests[eidstr].append(
# skip top-most tb in sweep_args
(argname, sargvalue, tb.tb_next, estr))
if __debug__:
msg = "%s on %s=%s" % (estr, argname, argvalue)
debug('TEST',
'Failed unittest: %s\n%s' % (eidstr, msg))
if report_progress:
sys.stdout.write(status)
sys.stdout.flush()
untrain_clf(argvalue)
# TODO: handle different levels of unittests properly
if cfg.getboolean('tests', 'quick', False):
# on TESTQUICK just run test for 1st entry in the list,
# the rest are omitted
# TODO: proper partitioning of unittests
break
