def select_tests(all_tests, test_categories, selected_categories, prefix=''):
'''Return the set of tests contained by the selected test categories.'''
test_categories['_all_'] = [test.path for test in all_tests]
if ('_default_' in selected_categories
and '_default_' not in test_categories):
selected_categories = ['_all_']
# Recursively expand job categories.
while compat.compat_any(cat in test_categories
for cat in selected_categories):
tmp = []
for cat in selected_categories:
if cat in test_categories:
tmp.extend(test_categories[cat])
else:
# cat has been fully expanded and now refers to a test
# contained within the directory named cat.
tmp.append(cat)
selected_categories = tmp
# Select tests to run.
tests = []
parent = lambda pdir, cdir: \
not os.path.relpath(cdir, start=pdir).startswith(os.pardir)
for cat in selected_categories:
# test paths are relative to the config directory but absolute paths
# are stored .
found = False
cat_paths = glob.glob(os.path.join(prefix, cat))
for test in all_tests:
if cat == test.name:
found = True
tests.append(test)
elif compat.compat_any(
os.path.exists(path) and os.path.samefile(path, test.path)
for path in cat_paths):
found = True
tests.append(test)
elif compat.compat_any(
parent(path, test.path) for path in cat_paths):
# test contained within a subdirectory of a cat_path.
found = True
tests.append(test)
if not found:
print('WARNING: %s test/category not found.\n' % cat)
# Only want to run each test once.
tests = list(compat.compat_set(tests))
return tests
def select_tests(all_tests, test_categories, selected_categories, prefix=''):
'''Return the set of tests contained by the selected test categories.'''
test_categories['_all_'] = [test.path for test in all_tests]
if ('_default_' in selected_categories
and '_default_' not in test_categories):
selected_categories = ['_all_']
# Recursively expand job categories.
while compat.compat_any(
cat in test_categories for cat in selected_categories
):
tmp = []
for cat in selected_categories:
if cat in test_categories:
tmp.extend(test_categories[cat])
else:
# cat has been fully expanded and now refers to a test
# contained within the directory named cat.
tmp.append(cat)
selected_categories = tmp
# Select tests to run.
tests = []
parent = lambda pdir, cdir: \
not os.path.relpath(cdir, start=pdir).startswith(os.pardir)
for cat in selected_categories:
# test paths are relative to the config directory but absolute paths
# are stored .
found = False
cat_paths = glob.glob(os.path.join(prefix, cat))
for test in all_tests:
if cat == test.name:
found = True
tests.append(test)
elif compat.compat_any(os.path.exists(path) and
os.path.samefile(path, test.path) for path in cat_paths):
found = True
tests.append(test)
elif compat.compat_any(parent(path, test.path)
for path in cat_paths):
# test contained within a subdirectory of a cat_path.
found = True
tests.append(test)
if not found:
print('WARNING: %s test/category not found.\n' % cat)
# Only want to run each test once.
tests = list(compat.compat_set(tests))
return tests
def __init__(self, bools=None, status=None, name=None):
(self._unknown, self._skipped) = (-2, -1)
(self._passed, self._partial, self._failed) = (0, 1, 2)
if name is not None:
setattr(self, 'status', getattr(self, '_' + name))
elif status is not None:
self.status = status
elif bools:
if compat.compat_all(bools):
self.status = self._passed
elif compat.compat_any(bools):
self.status = self._partial
else:
self.status = self._failed
else:
self.status = self._unknown
def __init__(self, bools=None, status=None, name=None):
(self._unknown, self._skipped) = (-2, -1)
(self._passed, self._partial, self._failed) = (0, 1, 2)
if name is not None:
setattr(self, 'status', getattr(self, '_'+name))
elif status is not None:
self.status = status
elif bools:
if compat.compat_all(bools):
self.status = self._passed
elif compat.compat_any(bools):
self.status = self._partial
else:
self.status = self._failed
else:
self.status = self._unknown
def run_test(self, verbose=1, cluster_queue=None, rundir=None):
'''Run all jobs in test.'''
try:
# Construct tests.
test_cmds = []
test_files = []
for (test_input, test_arg) in self.inputs_args:
if (test_input and not os.path.exists(
os.path.join(self.path, test_input))):
err = 'Input file does not exist: %s' % (test_input, )
raise exceptions.RunError(err)
test_cmds.append(
self.test_program.run_cmd(test_input, test_arg,
self.nprocs))
test_files.append(
util.testcode_filename(FILESTEM['test'],
self.test_program.test_id,
test_input, test_arg))
# Move files matching output pattern out of the way.
self.move_old_output_files(verbose)
# Run tests one-at-a-time locally or submit job in single submit
# file to a queueing system.
if cluster_queue:
if self.output:
for (ind, test) in enumerate(test_cmds):
# Don't quote self.output if it contains any wildcards
# (assume the user set it up correctly!)
out = self.output
if not compat.compat_any(
wild in self.output
for wild in ['*', '?', '[', '{']):
out = pipes.quote(self.output)
test_cmds[ind] = '%s; mv %s %s' % (
test_cmds[ind], out, pipes.quote(test_files[ind]))
test_cmds = ['\n'.join(test_cmds)]
for (ind, test) in enumerate(test_cmds):
job = self.start_job(test, cluster_queue, verbose)
job.wait()
# Analyse tests as they finish.
if cluster_queue:
# Did all of them at once.
for (test_input, test_arg) in self.inputs_args:
self.verify_job(test_input, test_arg, verbose, rundir)
else:
# Did one job at a time.
(test_input, test_arg) = self.inputs_args[ind]
err = []
if self.output:
try:
self.move_output_to_test_output(test_files[ind])
except exceptions.RunError:
err.append(sys.exc_info()[1])
status = validation.Status()
if job.returncode != 0:
err.insert(
0, 'Error running job. Return code: %i' %
job.returncode)
(status, msg) = self.skip_job(test_input, test_arg,
verbose)
if status.skipped():
self._update_status(status, (test_input, test_arg))
if verbose > 0 and verbose < 3:
sys.stdout.write(
util.info_line(self.path, test_input, test_arg,
rundir))
status.print_status(msg, verbose)
elif err:
# re-raise first error we hit.
raise exceptions.RunError(err[0])
else:
self.verify_job(test_input, test_arg, verbose, rundir)
except exceptions.RunError:
err = sys.exc_info()[1]
if verbose > 2:
err = 'Test(s) in %s failed.\n%s' % (self.path, err)
status = validation.Status([False])
self._update_status(status, (test_input, test_arg))
if verbose > 0 and verbose < 3:
info_line = util.info_line(self.path, test_input, test_arg,
rundir)
sys.stdout.write(info_line)
status.print_status(err, verbose)
# Shouldn't run remaining tests after such a catastrophic failure.
# Mark all remaining tests as skipped so the user knows that they
# weren't run.
err = 'Previous test in %s caused a system failure.' % (self.path)
status = validation.Status(name='skipped')
for ((test_input, test_arg), stat) in self.status.items():
if not self.status[(test_input, test_arg)]:
self._update_status(status, (test_input, test_arg))
if verbose > 2:
cmd = self.test_program.run_cmd(
test_input, test_arg, self.nprocs)
print('Test using %s in %s' % (cmd, self.path))
elif verbose > 0:
info_line = util.info_line(self.path, test_input,
test_arg, rundir)
sys.stdout.write(info_line)
status.print_status(err, verbose)
def run_test(self, verbose=1, cluster_queue=None, rundir=None):
'''Run all jobs in test.'''
try:
# Construct tests.
test_cmds = []
test_files = []
for (test_input, test_arg) in self.inputs_args:
if (test_input and
not os.path.exists(os.path.join(self.path,test_input))):
err = 'Input file does not exist: %s' % (test_input,)
raise exceptions.RunError(err)
test_cmds.append(self.test_program.run_cmd(test_input, test_arg,
self.nprocs))
test_files.append(util.testcode_filename(FILESTEM['test'],
self.test_program.test_id, test_input, test_arg))
# Move files matching output pattern out of the way.
self.move_old_output_files(verbose)
# Run tests one-at-a-time locally or submit job in single submit
# file to a queueing system.
if cluster_queue:
if self.output:
for (ind, test) in enumerate(test_cmds):
# Don't quote self.output if it contains any wildcards
# (assume the user set it up correctly!)
out = self.output
if not compat.compat_any(wild in self.output for wild in
['*', '?', '[', '{']):
out = pipes.quote(self.output)
test_cmds[ind] = '%s; mv %s %s' % (test_cmds[ind],
out, pipes.quote(test_files[ind]))
test_cmds = ['\n'.join(test_cmds)]
for (ind, test) in enumerate(test_cmds):
job = self.start_job(test, cluster_queue, verbose)
job.wait()
# Analyse tests as they finish.
if cluster_queue:
# Did all of them at once.
for (test_input, test_arg) in self.inputs_args:
self.verify_job(test_input, test_arg, verbose, rundir)
else:
# Did one job at a time.
(test_input, test_arg) = self.inputs_args[ind]
err = []
if self.output:
try:
self.move_output_to_test_output(test_files[ind])
except exceptions.RunError:
err.append(sys.exc_info()[1])
status = validation.Status()
if job.returncode != 0:
err.insert(0, 'Error running job. Return code: %i'
% job.returncode)
(status, msg) = self.skip_job(test_input, test_arg,
verbose)
if status.skipped():
self._update_status(status, (test_input, test_arg))
if verbose > 0 and verbose < 3:
sys.stdout.write(
util.info_line(self.path,
test_input, test_arg, rundir)
)
status.print_status(msg, verbose)
elif err:
# re-raise first error we hit.
raise exceptions.RunError(err[0])
else:
self.verify_job(test_input, test_arg, verbose, rundir)
except exceptions.RunError:
err = sys.exc_info()[1]
if verbose > 2:
err = 'Test(s) in %s failed.\n%s' % (self.path, err)
status = validation.Status([False])
self._update_status(status, (test_input, test_arg))
if verbose > 0 and verbose < 3:
info_line = util.info_line(self.path, test_input, test_arg, rundir)
sys.stdout.write(info_line)
status.print_status(err, verbose)
# Shouldn't run remaining tests after such a catastrophic failure.
# Mark all remaining tests as skipped so the user knows that they
# weren't run.
err = 'Previous test in %s caused a system failure.' % (self.path)
status = validation.Status(name='skipped')
for ((test_input, test_arg), stat) in self.status.items():
if not self.status[(test_input,test_arg)]:
self._update_status(status, (test_input, test_arg))
if verbose > 2:
cmd = self.test_program.run_cmd(test_input, test_arg,
self.nprocs)
print('Test using %s in %s' % (cmd, self.path))
elif verbose > 0:
info_line = util.info_line(self.path, test_input,
test_arg, rundir)
sys.stdout.write(info_line)
status.print_status(err, verbose)
