def create_test_suites(filename=None, config=None, _globals=None, options=None):
if options is None: #pragma:nocover
options = Options()
#
# Add categories specified by the PYUTILIB_AUTOTEST_CATEGORIES
# or PYUTILIB_UNITTEST_CATEGORIES environments
#
if options is None or options.categories is None or len(
options.categories) == 0:
options.categories = set()
if 'PYUTILIB_AUTOTEST_CATEGORIES' in os.environ:
for cat in re.split(',',
os.environ['PYUTILIB_AUTOTEST_CATEGORIES']):
if cat != '':
options.categories.add(cat.strip())
elif 'PYUTILIB_UNITTEST_CATEGORIES' in os.environ:
for cat in re.split(',',
os.environ['PYUTILIB_UNITTEST_CATEGORIES']):
if cat != '':
options.categories.add(cat.strip())
#
if not filename is None:
if options.currdir is None:
options.currdir = dirname(abspath(filename)) + os.sep
#
ep = ExtensionPoint(plugins.ITestParser)
ftype = os.path.splitext(filename)[1]
if not ftype == '':
ftype = ftype[1:]
service = ep.service(ftype)
if service is None:
raise IOError(
"Unknown file type. Cannot load test configuration from file '%s'"
% filename)
config = service.load_test_config(filename)
#service.print_test_config(config)
validate_test_config(config)
#
# Evaluate Python expressions
#
for item in config.get('python', []):
try:
exec(item, _globals)
except Exception:
err = sys.exc_info()[1]
print("ERROR executing '%s'" % item)
print(" Exception: %s" % str(err))
#
# Create test driver, which is put in the global namespace
#
driver = plugins.TestDriverFactory(config['driver'])
if driver is None:
raise IOError("Unexpected test driver '%s'" % config['driver'])
_globals["test_driver"] = driver
#
# Generate suite
#
for suite in config.get('suites', {}):
create_test_suite(suite, config, _globals, options)
def create_test_suite(suite, config, _globals, options):
#
# Skip suite creation if the options categores do not intersect with the list of test suite categories
#
if len(options.categories) > 0:
flag = False
for cat in options.categories:
if cat in config['suites'][suite].get('categories',[]):
flag = True
break
if not flag:
return
#
# Create test driver
#
if suite in _globals:
raise IOError("Cannot create suite '%s' since there is another symbol with that name in the global namespace!" % suite)
def setUpClassFn(cls):
options = cls._options[None]
cls._test_driver.setUpClass(cls,options)
_globals[suite] = type(str(suite),(unittest.TestCase,),{'setUpClass': classmethod(setUpClassFn)})
_globals[suite]._options[None] = options
setattr(_globals[suite],'_test_driver', _globals['test_driver'])
setattr(_globals[suite],'suite_categories', config['suites'][suite].get('categories',[]))
#
# Create test functions
#
tests = []
if 'tests' in config['suites'][suite]:
for item in config['suites'][suite]['tests']:
tests.append( (item['solver'], item['problem'], item) )
else:
for solver in config['suites'][suite]['solvers']:
for problem in config['suites'][suite]['problems']:
tests.append( (solver, problem, {}) )
#
for solver, problem, item in tests:
##sname = solver
if options.testname_format is None:
test_name = solver+"_"+problem
else:
test_name = options.testname_format % (solver, problem)
#
def fn(testcase, name, suite):
options = testcase._options[suite,name]
fn.test_driver.setUp(testcase, options)
ans = fn.test_driver.run_test(testcase, name, options)
fn.test_driver.tearDown(testcase, options)
return ans
fn.test_driver = _globals['test_driver']
#
_options = Options()
#
problem_options = config['suites'][suite]['problems'][problem]
if not problem_options is None and 'problem' in problem_options:
_problem = problem_options['problem']
else:
_problem = problem
for attr,value in config['problems'].get(_problem,{}).items():
_options[attr] = _str(value)
if not problem_options is None:
for attr,value in problem_options.items():
_options[attr] = _str(value)
#
solver_options = config['suites'][suite]['solvers'][solver]
if not solver_options is None and 'solver' in solver_options:
_solver = solver_options['solver']
else:
_solver = solver
_name = _solver
for attr,value in config['solvers'].get(_solver,{}).items():
_options[attr] = _str(value)
if attr == 'name':
_name = value
if not solver_options is None:
for attr,value in solver_options.items():
_options[attr] = _str(value)
#
for key in item:
if key not in ['problem','solver']:
_options[key] = _str(item[key])
#
_options.solver = _str(_name)
_options.problem = _str(_problem)
_options.suite = _str(suite)
_options.currdir = _str(options.currdir)
#
_globals[suite].add_fn_test(name=test_name, fn=fn, suite=suite, options=_options)
def create_test_suite(suite, config, _globals, options):
#
# Skip suite creation if the options categores do not intersect with the list of test suite categories
#
if len(options.categories) > 0:
flag = False
for cat in options.categories:
if cat in config['suites'][suite].get('categories', []):
flag = True
break
if not flag:
return
#
# Create test driver
#
if suite in _globals:
raise IOError(
"Cannot create suite '%s' since there is another symbol with that name in the global namespace!"
% suite)
def setUpClassFn(cls):
options = cls._options[None]
cls._test_driver.setUpClass(cls, options)
_globals[suite] = type(
str(suite),
(unittest.TestCase,), {'setUpClass': classmethod(setUpClassFn)})
_globals[suite]._options[None] = options
setattr(_globals[suite], '_test_driver', _globals['test_driver'])
setattr(_globals[suite], 'suite_categories', config['suites'][suite].get(
'categories', []))
#
# Create test functions
#
tests = []
if 'tests' in config['suites'][suite]:
for item in config['suites'][suite]['tests']:
tests.append((item['solver'], item['problem'], item))
else:
for solver in config['suites'][suite]['solvers']:
for problem in config['suites'][suite]['problems']:
tests.append((solver, problem, {}))
#
for solver, problem, item in tests:
##sname = solver
if options.testname_format is None:
test_name = solver + "_" + problem
else:
test_name = options.testname_format % (solver, problem)
#
def fn(testcase, name, suite):
options = testcase._options[suite, name]
fn.test_driver.setUp(testcase, options)
ans = fn.test_driver.run_test(testcase, name, options)
fn.test_driver.tearDown(testcase, options)
return ans
fn.test_driver = _globals['test_driver']
#
_options = Options()
#
problem_options = config['suites'][suite]['problems'][problem]
if not problem_options is None and 'problem' in problem_options:
_problem = problem_options['problem']
else:
_problem = problem
for attr, value in config['problems'].get(_problem, {}).items():
_options[attr] = _str(value)
if not problem_options is None:
for attr, value in problem_options.items():
_options[attr] = _str(value)
#
solver_options = config['suites'][suite]['solvers'][solver]
if not solver_options is None and 'solver' in solver_options:
_solver = solver_options['solver']
else:
_solver = solver
_name = _solver
for attr, value in config['solvers'].get(_solver, {}).items():
_options[attr] = _str(value)
if attr == 'name':
_name = value
if not solver_options is None:
for attr, value in solver_options.items():
_options[attr] = _str(value)
#
for key in item:
if key not in ['problem', 'solver']:
_options[key] = _str(item[key])
#
_options.solver = _str(_name)
_options.problem = _str(_problem)
_options.suite = _str(suite)
_options.currdir = _str(options.currdir)
#
_globals[suite].add_fn_test(
name=test_name, fn=fn, suite=suite, options=_options)