def run(limit=None, verbosity=None, exit_=False):
"""Runs the full or a subset of the PyMVPA unittest suite.
Parameters
----------
limit : None or list
If None, the full test suite is run. Alternatively, a list with test IDs
can be provides. IDs are the base filenames of the test implementation,
e.g. the ID for the suite in 'mvpa/tests/test_niftidataset.py' is
'niftidataset'.
verbosity : None or int
Verbosity of unittests execution. If None, controlled by PyMVPA
configuration tests/verbosity. Values >=3 enable all Python,
and PyMVPA warnings, >=4 adds NumPy warnings, >=5 -- nose debug info.
exit_ : bool, optional
Either to exit with an error code upon the completion.
"""
if __debug__:
from mvpa.base import debug
# Lets add some targets which provide additional testing
debug.active += ['CHECK_.*']
if verbosity is None:
verbosity = int(cfg.get('tests', 'verbosity', default=1))
# provide people with a hint about the warnings that might show up in a
# second
if verbosity:
print("T: MVPA_SEED=%s" % _random_seed)
if verbosity > 1:
print('T: Testing for availability of external software packages.')
# So we could see all warnings about missing dependencies
maxcount = warning.maxcount
warning.maxcount = 1000
# fully test of externals
externals.test_all_dependencies(verbosity=max(0, verbosity-1))
if verbosity < 3:
# no MVPA warnings during whole testsuite (but restore handlers later on)
handler_backup = warning.handlers
warning.handlers = []
# No python warnings (like ctypes version for slmr)
import warnings
warnings.simplefilter('ignore')
if verbosity < 4:
# No NumPy
np_errsettings = np.geterr()
np.seterr(**dict([(x, 'ignore') for x in np_errsettings]))
try:
if externals.exists('nose'):
# Lets just use nose
run_tests_using_nose(limit=limit,
verbosity=verbosity,
exit_=exit_)
else:
print("T: Warning -- major bulk of tests is skipped since nose "
"is unavailable")
# collect all tests
suites = collect_test_suites(verbosity=verbosity)
if limit is None:
# make global test suite (use them all)
ts = unittest.TestSuite(suites.values())
else:
ts = unittest.TestSuite([suites[s] for s in limit])
class TextTestRunnerPyMVPA(unittest.TextTestRunner):
"""Extend TextTestRunner to print out random seed which was
used in the case of failure"""
def run(self, test):
"""Run the bloody test and puke the seed value if failed"""
result = super(TextTestRunnerPyMVPA, self).run(test)
if not result.wasSuccessful():
print "MVPA_SEED=%s" % _random_seed
# finally run it
TextTestRunnerPyMVPA(verbosity=verbosity).run(ts)
finally:
# restore warning handlers
warning.maxcount = maxcount
if verbosity < 3:
# restore warning handlers
warning.handlers = handler_backup
if verbosity < 4:
# restore numpy settings
np.seterr(**np_errsettings)
def collect_test_suites():
"""Runs over all tests it knows and composes a dictionary with test suite
instances as values and IDs as keys. IDs are the filenames of the unittest
without '.py' extension and 'test_' prefix.
During collection this function will run a full and verbose test for all
known externals.
"""
# list all test modules (without .py extension)
tests = [
# Basic data structures/manipulators
'test_externals',
'test_base',
'test_dochelpers',
'test_som',
'test_splitter',
'test_state',
'test_params',
# Misc supporting utilities
'test_config',
'test_stats',
'test_support',
'test_verbosity',
'test_iohelpers',
'test_report',
'test_datasetfx',
'test_cmdline',
'test_args',
'test_meg',
# Classifiers (longer tests)
'test_clf',
'test_regr',
'test_knn',
'test_gnb',
'test_svm',
'test_plr',
'test_smlr',
# Various algorithms
'test_svdmapper',
'test_procrust',
'test_hyperalignment',
'test_transformers',
'test_clfcrossval',
'test_searchlight',
'test_rfe',
'test_ifs',
'test_datameasure',
'test_perturbsensana',
# And the suite (all-in-1)
'test_suite',
]
# provide people with a hint about the warnings that might show up in a
# second
warning('Testing for availability of external software packages. Test '
'cases depending on missing packages will not be part of the test '
'suite.')
# So we could see all warnings about missing dependencies
warning.maxcount = 1000
# fully test of externals
externals.test_all_dependencies()
__optional_tests = [ ('scipy', 'ridge'),
('scipy', 'stats_sp'),
('scipy', 'gpr'),
(['lars','scipy'], 'lars'),
(['cPickle', 'gzip'], 'hamster'),
]
# and now for the optional tests
optional_tests = []
for external, testname in __optional_tests:
if externals.exists(external):
optional_tests.append('test_%s' % testname)
# finally merge all of them
tests += optional_tests
# import all test modules
for t in tests:
exec 'import ' + t
# instantiate all tests suites and return dict of them (with ID as key)
return dict([(t[5:], eval(t + '.suite()')) for t in tests ])
def run(limit=None, verbosity=None, exit_=False):
"""Runs the full or a subset of the PyMVPA unittest suite.
Parameters
----------
limit : None or list
If None, the full test suite is run. Alternatively, a list with test IDs
can be provides. IDs are the base filenames of the test implementation,
e.g. the ID for the suite in 'mvpa/tests/test_niftidataset.py' is
'niftidataset'.
verbosity : None or int
Verbosity of unittests execution. If None, controlled by PyMVPA
configuration tests/verbosity
exit_ : bool, optional
Either to exit with an error code upon the completion.
"""
if __debug__:
from mvpa.base import debug
# Lets add some targets which provide additional testing
debug.active += ['CHECK_.*']
if verbosity is None:
verbosity = int(cfg.get('tests', 'verbosity', default=1))
# provide people with a hint about the warnings that might show up in a
# second
if verbosity:
print("T: MVPA_SEED=%s" % _random_seed)
if verbosity > 1:
print('T: Testing for availability of external software packages.')
# So we could see all warnings about missing dependencies
warning.maxcount = 1000
# fully test of externals
externals.test_all_dependencies(verbosity=max(0, verbosity - 1))
# no MVPA warnings during whole testsuite (but restore handlers later on)
handler_backup = warning.handlers
warning.handlers = []
# No python warnings (like ctypes version for slmr)
import warnings
warnings.simplefilter('ignore')
try:
if externals.exists('nose'):
# Lets just use nose
run_tests_using_nose(limit=limit, verbosity=verbosity, exit_=exit_)
else:
print(
"T: Warning -- major bulk of tests is skipped since nose "
"is unavailable")
# collect all tests
suites = collect_test_suites(verbosity=verbosity)
if limit is None:
# make global test suite (use them all)
ts = unittest.TestSuite(suites.values())
else:
ts = unittest.TestSuite([suites[s] for s in limit])
class TextTestRunnerPyMVPA(unittest.TextTestRunner):
"""Extend TextTestRunner to print out random seed which was
used in the case of failure"""
def run(self, test):
"""Run the bloody test and puke the seed value if failed"""
result = super(TextTestRunnerPyMVPA, self).run(test)
if not result.wasSuccessful():
print "MVPA_SEED=%s" % _random_seed
if verbosity is None:
verbosity = int(cfg.get('tests', 'verbosity', default=1))
# finally run it
TextTestRunnerPyMVPA(verbosity=verbosity).run(ts)
finally:
# restore warning handlers
warning.handlers = handler_backup
