def run_cmd(self, input_file, args, nprocs=0):
'''Create run command.'''
output_file = util.testcode_filename(FILESTEM['test'], self.test_id,
input_file, args)
error_file = util.testcode_filename(FILESTEM['error'], self.test_id,
input_file, args)
# Need to escape filenames for passing them to the shell.
exe = pipes.quote(self.exe)
output_file = pipes.quote(output_file)
error_file = pipes.quote(error_file)
cmd = self.run_cmd_template.replace('tc.program', exe)
if type(input_file) is str:
input_file = pipes.quote(input_file)
cmd = cmd.replace('tc.input', input_file)
else:
cmd = cmd.replace('tc.input', '')
if type(args) is str:
cmd = cmd.replace('tc.args', args)
else:
cmd = cmd.replace('tc.args', '')
cmd = cmd.replace('tc.output', output_file)
cmd = cmd.replace('tc.error', error_file)
if nprocs > 0 and self.launch_parallel:
cmd = '%s %s' % (self.launch_parallel, cmd)
cmd = cmd.replace('tc.nprocs', str(nprocs))
return cmd
def create_new_benchmarks(self, benchmark, copy_files_since=None,
copy_files_path='testcode_data'):
'''Copy the test files to benchmark files.'''
oldcwd = os.getcwd()
os.chdir(self.path)
test_files = []
for (inp, arg) in self.inputs_args:
test_file = util.testcode_filename(FILESTEM['test'],
self.test_program.test_id, inp, arg)
err_file = util.testcode_filename(FILESTEM['error'],
self.test_program.test_id, inp, arg)
bench_file = util.testcode_filename(_FILESTEM_DICT['benchmark'],
benchmark, inp, arg)
test_files.extend((test_file, err_file, bench_file))
shutil.copy(test_file, bench_file)
if copy_files_since:
if not os.path.isdir(copy_files_path):
os.mkdir(copy_files_path)
if os.path.isdir(copy_files_path):
for data_file in glob.glob('*'):
if (os.path.isfile(data_file) and
os.stat(data_file)[-2] >= copy_files_since and
data_file not in test_files):
bench_data_file = os.path.join(copy_files_path,
data_file)
# shutil.copy can't overwrite files so remove old ones
# with the same name.
if os.path.exists(bench_data_file):
os.unlink(bench_data_file)
shutil.copy(data_file, bench_data_file)
os.chdir(oldcwd)
def create_new_benchmarks(self, benchmark, copy_files_since=None,
copy_files_path='testcode_data'):
'''Copy the test files to benchmark files.'''
oldcwd = os.getcwd()
os.chdir(self.path)
test_files = []
for (inp, arg) in self.inputs_args:
test_file = util.testcode_filename(FILESTEM['test'],
self.test_program.test_id, inp, arg)
err_file = util.testcode_filename(FILESTEM['error'],
self.test_program.test_id, inp, arg)
bench_file = util.testcode_filename(_FILESTEM_DICT['benchmark'],
benchmark, inp, arg)
test_files.extend((test_file, err_file, bench_file))
shutil.copy(test_file, bench_file)
if copy_files_since:
if not os.path.isdir(copy_files_path):
os.mkdir(copy_files_path)
if os.path.isdir(copy_files_path):
for data_file in glob.glob('*'):
if (os.path.isfile(data_file) and
os.stat(data_file)[-2] >= copy_files_since and
data_file not in test_files):
bench_data_file = os.path.join(copy_files_path,
data_file)
# shutil.copy can't overwrite files so remove old ones
# with the same name.
if os.path.exists(bench_data_file):
os.unlink(bench_data_file)
shutil.copy(data_file, bench_data_file)
os.chdir(oldcwd)
def run_cmd(self, input_file, args, nprocs=0):
'''Create run command.'''
output_file = util.testcode_filename(FILESTEM['test'], self.test_id,
input_file, args)
error_file = util.testcode_filename(FILESTEM['error'], self.test_id,
input_file, args)
# Need to escape filenames for passing them to the shell.
exe = pipes.quote(self.exe)
output_file = pipes.quote(output_file)
error_file = pipes.quote(error_file)
cmd = self.run_cmd_template.replace('tc.program', exe)
if type(input_file) is str:
input_file = pipes.quote(input_file)
cmd = cmd.replace('tc.input', input_file)
else:
cmd = cmd.replace('tc.input', '')
if type(args) is str:
cmd = cmd.replace('tc.args', args)
else:
cmd = cmd.replace('tc.args', '')
cmd = cmd.replace('tc.output', output_file)
cmd = cmd.replace('tc.error', error_file)
if nprocs > 0 and self.launch_parallel:
cmd = '%s %s' % (self.launch_parallel, cmd)
cmd = cmd.replace('tc.nprocs', str(nprocs))
return cmd
def extract_data(self, input_file, args, verbose=1):
'''Extract data from output file.
Assume function is executed in self.path.'''
tp_ptr = self.test_program
if tp_ptr.data_tag:
# Using internal data extraction function.
data_files = [
util.testcode_filename(FILESTEM['benchmark'],
tp_ptr.benchmark, input_file, args),
util.testcode_filename(FILESTEM['test'],
tp_ptr.test_id, input_file, args),
]
if verbose > 2:
print('Analysing output using data_tag %s in %s on files %s.' %
(tp_ptr.data_tag, self.path, ' and '.join(data_files)))
outputs = [util.extract_tagged_data(tp_ptr.data_tag, dfile)
for dfile in data_files]
else:
# Using external data extraction script.
# Get extraction commands.
extract_cmds = self.test_program.extract_cmd(input_file, args)
# Extract data.
outputs = []
for cmd in extract_cmds:
try:
if verbose > 2:
print('Analysing output using %s in %s.' %
(cmd, self.path))
extract_popen = subprocess.Popen(cmd, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
extract_popen.wait()
except OSError:
# slightly odd syntax in order to be compatible with python
# 2.5 and python 2.6/3
err = 'Analysing output failed: %s' % (sys.exc_info()[1],)
raise exceptions.AnalysisError(err)
# Convert data string from extract command to dictionary format.
if extract_popen.returncode != 0:
err = extract_popen.communicate()[1].decode('utf-8')
err = 'Analysing output failed: %s' % (err)
raise exceptions.AnalysisError(err)
data_string = extract_popen.communicate()[0].decode('utf-8')
if self.test_program.extract_fmt == 'table':
outputs.append(util.dict_table_string(data_string))
elif self.test_program.extract_fmt == 'yaml':
outputs.append({})
# convert values to be in a tuple so the format matches
# that from dict_table_string.
# ensure all keys are strings so they can be sorted
# (different data types cause problems!)
for (key, val) in yaml.safe_load(data_string).items():
if isinstance(val, list):
outputs[-1][str(key)] = tuple(val)
else:
outputs[-1][str(key)] = tuple((val,))
return tuple(outputs)
def skip_cmd(self, input_file, args):
'''Create skip command.'''
test_file = util.testcode_filename(FILESTEM['test'], self.test_id,
input_file, args)
error_file = util.testcode_filename(FILESTEM['error'], self.test_id,
input_file, args)
cmd = self.skip_cmd_template
cmd = cmd.replace('tc.skip', pipes.quote(self.skip_program))
cmd = cmd.replace('tc.args', self.skip_args)
cmd = cmd.replace('tc.test', pipes.quote(test_file))
cmd = cmd.replace('tc.error', pipes.quote(error_file))
return cmd
def skip_cmd(self, input_file, args):
'''Create skip command.'''
test_file = util.testcode_filename(FILESTEM['test'], self.test_id,
input_file, args)
error_file = util.testcode_filename(FILESTEM['error'], self.test_id,
input_file, args)
cmd = self.skip_cmd_template
cmd = cmd.replace('tc.skip', pipes.quote(self.skip_program))
cmd = cmd.replace('tc.args', self.skip_args)
cmd = cmd.replace('tc.test', pipes.quote(test_file))
cmd = cmd.replace('tc.error', pipes.quote(error_file))
return cmd
def extract_data(self, input_file, args, verbose=1):
'''Extract data from output file.
Assume function is executed in self.path.'''
tp_ptr = self.test_program
if tp_ptr.data_tag:
# Using internal data extraction function.
data_files = [
tp_ptr.select_benchmark_file(self.path, input_file, args),
util.testcode_filename(FILESTEM['test'],
tp_ptr.test_id, input_file, args),
]
if verbose > 2:
print(('Analysing output using data_tag %s in %s on files %s.' %
(tp_ptr.data_tag, self.path, ' and '.join(data_files))))
outputs = [util.extract_tagged_data(tp_ptr.data_tag, dfile)
for dfile in data_files]
else:
# Using external data extraction script.
# Get extraction commands.
extract_cmds = tp_ptr.extract_cmd(self.path, input_file, args)
# Extract data.
outputs = []
for cmd in extract_cmds:
try:
if verbose > 2:
print(('Analysing output using %s in %s.' %
(cmd, self.path)))
extract_popen = subprocess.run(cmd, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except OSError:
# slightly odd syntax in order to be compatible with python
# 2.5 and python 2.6/3
err = 'Analysing output failed: %s' % (sys.exc_info()[1],)
raise exceptions.AnalysisError(err)
# Convert data string from extract command to dictionary format.
if extract_popen.returncode != 0:
err = extract_popen.communicate()[1].decode('utf-8')
err = 'Analysing output failed: %s' % (err)
raise exceptions.AnalysisError(err)
data_string = extract_popen.stdout.decode('utf-8')
if self.test_program.extract_fmt == 'table':
outputs.append(util.dict_table_string(data_string))
elif self.test_program.extract_fmt == 'yaml':
outputs.append({})
# convert values to be in a tuple so the format matches
# that from dict_table_string.
# ensure all keys are strings so they can be sorted
# (different data types cause problems!)
for (key, val) in list(yaml.safe_load(data_string).items()):
if isinstance(val, list):
outputs[-1][str(key)] = tuple(val)
else:
outputs[-1][str(key)] = tuple((val,))
return tuple(outputs)
def extract_cmd(self, input_file, args):
'''Create extraction command(s).'''
test_file = util.testcode_filename(FILESTEM['test'], self.test_id,
input_file, args)
bench_file = util.testcode_filename(FILESTEM['benchmark'],
self.benchmark, input_file, args)
cmd = self.extract_cmd_template
cmd = cmd.replace('tc.extract', pipes.quote(self.extract_program))
cmd = cmd.replace('tc.args', self.extract_args)
if self.verify:
# Single command to compare benchmark and test outputs.
cmd = cmd.replace('tc.test', pipes.quote(test_file))
cmd = cmd.replace('tc.bench', pipes.quote(bench_file))
return (cmd, )
else:
# Need to return commands to extract data from the test and
# benchmark outputs.
test_cmd = cmd.replace('tc.file', pipes.quote(test_file))
bench_cmd = cmd.replace('tc.file', pipes.quote(bench_file))
return (bench_cmd, test_cmd)
def extract_cmd(self, input_file, args):
'''Create extraction command(s).'''
test_file = util.testcode_filename(FILESTEM['test'], self.test_id,
input_file, args)
bench_file = util.testcode_filename(FILESTEM['benchmark'],
self.benchmark, input_file, args)
cmd = self.extract_cmd_template
cmd = cmd.replace('tc.extract', pipes.quote(self.extract_program))
cmd = cmd.replace('tc.args', self.extract_args)
if self.verify:
# Single command to compare benchmark and test outputs.
cmd = cmd.replace('tc.test', pipes.quote(test_file))
cmd = cmd.replace('tc.bench', pipes.quote(bench_file))
return (cmd,)
else:
# Need to return commands to extract data from the test and
# benchmark outputs.
test_cmd = cmd.replace('tc.file', pipes.quote(test_file))
bench_cmd = cmd.replace('tc.file', pipes.quote(bench_file))
return (bench_cmd, test_cmd)
def select_benchmark_file(self, path, input_file, args):
'''Find the first benchmark file out of all benchmark IDs which exists.'''
benchmark = None
benchmarks = []
for bench_id in self.benchmark:
benchfile = util.testcode_filename(FILESTEM['benchmark'], bench_id,
input_file, args)
benchmarks.append(benchfile)
if os.path.exists(os.path.join(path, benchfile)):
benchmark = benchfile
break
if not benchmark:
err = 'No benchmark found in %s. Checked for: %s.'
raise exceptions.TestCodeError(err % (path, ', '.join(benchmarks)))
return benchmark
def select_benchmark_file(self, path, input_file, args):
'''Find the first benchmark file out of all benchmark IDs which exists.'''
benchmark = None
benchmarks = []
for bench_id in self.benchmark:
benchfile = util.testcode_filename(FILESTEM['benchmark'], bench_id,
input_file, args)
benchmarks.append(benchfile)
if os.path.exists(os.path.join(path, benchfile)):
benchmark = benchfile
break
if not benchmark:
err = 'No benchmark found in %s. Checked for: %s.'
raise exceptions.TestCodeError(err % (path, ', '.join(benchmarks)))
return benchmark
def extract_data(self, input_file, args, verbose=1):
'''Extract data from output file.
Assume function is executed in self.path.'''
tp_ptr = self.test_program
if tp_ptr.data_tag:
# Using internal data extraction function.
data_files = [
tp_ptr.select_benchmark_file(self.path, input_file, args),
util.testcode_filename(FILESTEM['test'], tp_ptr.test_id,
input_file, args),
]
if verbose > 2:
print('Analysing output using data_tag %s in %s on files %s.' %
(tp_ptr.data_tag, self.path, ' and '.join(data_files)))
outputs = [
util.extract_tagged_data(tp_ptr.data_tag, dfile)
for dfile in data_files
]
else:
# Using external data extraction script.
# Get extraction commands.
extract_cmds = tp_ptr.extract_cmd(self.path, input_file, args)
# Extract data.
outputs = []
for cmd in extract_cmds:
try:
if verbose > 2:
print('Analysing output using %s in %s.' %
(cmd, self.path))
# Samuel Ponce: Popen.wait() creates deadlock if the data is too large
# See documented issue for example in:
# https://docs.python.org/2/library/subprocess.html#subprocess.Popen.returncode
#
# Previous code that create deadlock:
#extract_popen = subprocess.Popen(cmd, shell=True,
# stdout=subprocess.PIPE, stderr=subprocess.PIPE)
#extract_popen.wait()
#
# New code (this might not be the best but work for me):
extract_popen = subprocess.Popen(cmd,
bufsize=1,
shell=True,
stdin=open(os.devnull),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
lines = []
for line in iter(extract_popen.stdout.readline, ''):
#print line,
lines.append(line)
except OSError:
# slightly odd syntax in order to be compatible with python
# 2.5 and python 2.6/3
err = 'Analysing output failed: %s' % (sys.exc_info()[1], )
raise exceptions.AnalysisError(err)
# Convert data string from extract command to dictionary format.
# SP: Because of the above change, the test below cannot be done:
#if extract_popen.returncode != 0:
# err = extract_popen.communicate()[1].decode('utf-8')
# err = 'Analysing output failed: %s' % (err)
# raise exceptions.AnalysisError(err)
#data_string = extract_popen.communicate()[0].decode('utf-8')
data_string = ''.join(lines)
if self.test_program.extract_fmt == 'table':
outputs.append(util.dict_table_string(data_string))
elif self.test_program.extract_fmt == 'yaml':
outputs.append({})
# convert values to be in a tuple so the format matches
# that from dict_table_string.
# ensure all keys are strings so they can be sorted
# (different data types cause problems!)
for (key, val) in yaml.safe_load(data_string).items():
if isinstance(val, list):
outputs[-1][str(key)] = tuple(val)
else:
outputs[-1][str(key)] = tuple((val, ))
return tuple(outputs)
def parse_jobconfig(config_file, user_options, test_programs, settings=None):
'''Parse the test configurations from the jobconfig file.
config_file: location of the jobconfig file, either relative or absolute.'''
if not os.path.exists(config_file):
raise exceptions.TestCodeError(
'Job configuration file %s does not exist.' % (config_file)
)
# paths to the test directories can be specified relative to the config
# file.
config_directory = os.path.dirname(os.path.abspath(config_file))
jobconfig = compat.configparser.RawConfigParser()
jobconfig.optionxform = str # Case sensitive file.
jobconfig.read(config_file)
# Alter config file with additional settings provided.
if settings:
for (section_key, section) in settings.items():
for (option_key, value) in section.items():
jobconfig.set(section_key, option_key, value)
# Parse job categories.
# Just store as list of test names for now.
if jobconfig.has_section('categories'):
test_categories = dict(jobconfig.items('categories'))
for (key, val) in test_categories.items():
test_categories[key] = val.split()
jobconfig.remove_section('categories')
else:
test_categories = {}
# Parse individual sections for tests.
# Note that sections/paths may contain globs and hence correspond to
# multiple tests.
# First, find out the tests each section corresponds to.
test_sections = []
for section in jobconfig.sections():
# Expand any globs in the path/section name and create individual Test
# objects for each one.
if jobconfig.has_option(section, 'path'):
path = os.path.join(config_directory,
jobconfig.get(section, 'path'))
jobconfig.remove_option('path')
globbed_tests = [(section, test_path)
for test_path in glob.glob(path)]
else:
path = os.path.join(config_directory, section)
globbed_tests = [(test_path, test_path)
for test_path in glob.glob(path)]
test_sections.append((section, globbed_tests))
test_sections.sort(key=lambda sec_info: len(sec_info[1]), reverse=True)
test_info = {}
for (section, globbed_tests) in test_sections:
test_dict = {}
# test program
if jobconfig.has_option(section, 'program'):
test_program = test_programs[jobconfig.get(section, 'program')]
else:
test_program = test_programs[user_options['default_program']]
# tolerances
if jobconfig.has_option(section, 'tolerance'):
test_dict['tolerances'] = {}
for item in (
eval_nested_tuple(jobconfig.get(section,'tolerance'))
):
(name, tol) = parse_tolerance_tuple(item)
test_dict['tolerances'][name] = tol
jobconfig.remove_option(section, 'tolerance')
if None in test_dict['tolerances']:
test_dict['default_tolerance'] = test_dict['tolerances'][None]
# inputs and arguments
if jobconfig.has_option(section, 'inputs_args'):
# format: (input, arg), (input, arg)'
test_dict['inputs_args'] = (
eval_nested_tuple(jobconfig.get(section, 'inputs_args')))
jobconfig.remove_option(section, 'inputs_args')
if jobconfig.has_option(section, 'run_concurrent'):
test_dict['run_concurrent'] = \
jobconfig.getboolean(section, 'run_concurrent')
jobconfig.remove_option(section, 'run_concurrent')
# Other options.
for option in jobconfig.options(section):
test_dict[option] = jobconfig.get(section, option)
for key in ('nprocs', 'max_nprocs', 'min_nprocs'):
if key in test_dict:
test_dict[key] = int(test_dict[key])
for (name, path) in globbed_tests:
# Need to take care with tolerances: want to *update* existing
# tolerance dictionary rather than overwrite it.
# This means we can't just use test_dict to update the relevant
# dictionary in test_info.
tol = None
if name in test_info:
# Just update existing info.
test = test_info[name]
if 'tolerances' in test_dict:
test[2]['tolerances'].update(test_dict['tolerances'])
tol = test_dict.pop('tolerances')
test[0] = test_program
test[1] = path
test[2].update(test_dict)
if tol:
test_dict['tolerances'] = tol
else:
# Create new test_info value.
# Merge with default values.
# Default test options.
default_test = test_program.default_test_settings
test = dict(
inputs_args=default_test.inputs_args,
output=default_test.output,
default_tolerance=default_test.default_tolerance,
tolerances = copy.deepcopy(default_test.tolerances),
nprocs=default_test.nprocs,
min_nprocs=default_test.min_nprocs,
max_nprocs=default_test.max_nprocs,
run_concurrent=default_test.run_concurrent,
)
if 'tolerances' in test_dict:
test['tolerances'].update(test_dict['tolerances'])
tol = test_dict.pop('tolerances')
test.update(test_dict)
# restore tolerances for next test in the glob.
if tol:
test_dict['tolerances'] = tol
test_info[name] = [test_program, path, copy.deepcopy(test)]
# Now create the tests (after finding out what the input files are).
tests = []
for (name, (test_program, path, test_dict)) in test_info.items():
old_dir = os.getcwd()
os.chdir(path)
# Expand any globs in the input files.
inputs_args = []
for input_arg in test_dict['inputs_args']:
# Be a little forgiving for the input_args config option.
# If we're given ('input'), then clearly the user meant for the
# args option to be empty. However, literal_eval returns
# a string rather than a tuple in such cases, which causes
# problems.
if isinstance(input_arg, str):
inp = input_arg
arg = ''
elif len(input_arg) == 2:
inp = input_arg[0]
arg = input_arg[1]
else:
inp = input_arg[0]
arg = ''
if inp:
# the test, error and benchmark filenames contain the input
# filename, so we need to filter them out.
inp_files = sorted(glob.glob(inp))
if not inp_files:
err = 'Cannot find input file %s in %s.' % (inp, path)
raise exceptions.TestCodeError(err)
for inp_file in inp_files:
# We use a glob for the input argument to avoid the
# case where the argument is empty and hence a pattern
# such as *.inp also matches files like
# test.out.test_id.inp=x.inp and hence considering
# previous output files to actually be an input file in
# their own right.
test_files = [
util.testcode_filename(stem[1], '*', '*', arg)
for stem in testcode2._FILESTEM_TUPLE
]
testcode_files = []
for tc_file in test_files:
testcode_files.extend(glob.glob(tc_file))
if inp_file not in testcode_files:
inputs_args.append((inp_file, arg))
else:
inputs_args.append((inp, arg))
test_dict['inputs_args'] = tuple(inputs_args)
os.chdir(old_dir)
# Create test.
if test_dict['run_concurrent']:
for input_arg in test_dict['inputs_args']:
test_dict['inputs_args'] = (input_arg,)
tests.append(testcode2.Test(name, test_program, path,
**test_dict))
else:
tests.append(testcode2.Test(name, test_program, path, **test_dict))
return (tests, test_categories)
def run_test(self, verbose=1, cluster_queue=None, rundir=None):
'''Run all jobs in test.'''
try:
# Construct tests.
test_cmds = []
test_files = []
for (test_input, test_arg) in self.inputs_args:
if (test_input and
not os.path.exists(os.path.join(self.path,test_input))):
err = 'Input file does not exist: %s' % (test_input,)
raise exceptions.RunError(err)
test_cmds.append(self.test_program.run_cmd(test_input, test_arg,
self.nprocs))
test_files.append(util.testcode_filename(FILESTEM['test'],
self.test_program.test_id, test_input, test_arg))
# Move files matching output pattern out of the way.
self.move_old_output_files(verbose)
# Run tests one-at-a-time locally or submit job in single submit
# file to a queueing system.
if cluster_queue:
if self.output:
for (ind, test) in enumerate(test_cmds):
# Don't quote self.output if it contains any wildcards
# (assume the user set it up correctly!)
out = self.output
if not compat.compat_any(wild in self.output for wild in
['*', '?', '[', '{']):
out = pipes.quote(self.output)
test_cmds[ind] = '%s; mv %s %s' % (test_cmds[ind],
out, pipes.quote(test_files[ind]))
test_cmds = ['\n'.join(test_cmds)]
for (ind, test) in enumerate(test_cmds):
job = self.start_job(test, cluster_queue, verbose)
job.wait()
# Analyse tests as they finish.
if cluster_queue:
# Did all of them at once.
for (test_input, test_arg) in self.inputs_args:
self.verify_job(test_input, test_arg, verbose, rundir)
else:
# Did one job at a time.
(test_input, test_arg) = self.inputs_args[ind]
err = []
if self.output:
try:
self.move_output_to_test_output(test_files[ind])
except exceptions.RunError:
err.append(sys.exc_info()[1])
status = validation.Status()
if job.returncode != 0:
err.insert(0, 'Error running job. Return code: %i'
% job.returncode)
(status, msg) = self.skip_job(test_input, test_arg,
verbose)
if status.skipped():
self._update_status(status, (test_input, test_arg))
if verbose > 0 and verbose < 3:
sys.stdout.write(
util.info_line(self.path,
test_input, test_arg, rundir)
)
status.print_status(msg, verbose)
elif err:
# re-raise first error we hit.
raise exceptions.RunError(err[0])
else:
self.verify_job(test_input, test_arg, verbose, rundir)
except exceptions.RunError:
err = sys.exc_info()[1]
if verbose > 2:
err = 'Test(s) in %s failed.\n%s' % (self.path, err)
status = validation.Status([False])
self._update_status(status, (test_input, test_arg))
if verbose > 0 and verbose < 3:
info_line = util.info_line(self.path, test_input, test_arg, rundir)
sys.stdout.write(info_line)
status.print_status(err, verbose)
# Shouldn't run remaining tests after such a catastrophic failure.
# Mark all remaining tests as skipped so the user knows that they
# weren't run.
err = 'Previous test in %s caused a system failure.' % (self.path)
status = validation.Status(name='skipped')
for ((test_input, test_arg), stat) in self.status.items():
if not self.status[(test_input,test_arg)]:
self._update_status(status, (test_input, test_arg))
if verbose > 2:
cmd = self.test_program.run_cmd(test_input, test_arg,
self.nprocs)
print('Test using %s in %s' % (cmd, self.path))
elif verbose > 0:
info_line = util.info_line(self.path, test_input,
test_arg, rundir)
sys.stdout.write(info_line)
status.print_status(err, verbose)
def run_test(self, verbose=1, cluster_queue=None, rundir=None):
'''Run all jobs in test.'''
try:
# Construct tests.
test_cmds = []
test_files = []
for (test_input, test_arg) in self.inputs_args:
if (test_input and not os.path.exists(
os.path.join(self.path, test_input))):
err = 'Input file does not exist: %s' % (test_input, )
raise exceptions.RunError(err)
test_cmds.append(
self.test_program.run_cmd(test_input, test_arg,
self.nprocs))
test_files.append(
util.testcode_filename(FILESTEM['test'],
self.test_program.test_id,
test_input, test_arg))
# Move files matching output pattern out of the way.
self.move_old_output_files(verbose)
# Run tests one-at-a-time locally or submit job in single submit
# file to a queueing system.
if cluster_queue:
if self.output:
for (ind, test) in enumerate(test_cmds):
# Don't quote self.output if it contains any wildcards
# (assume the user set it up correctly!)
out = self.output
if not compat.compat_any(
wild in self.output
for wild in ['*', '?', '[', '{']):
out = pipes.quote(self.output)
test_cmds[ind] = '%s; mv %s %s' % (
test_cmds[ind], out, pipes.quote(test_files[ind]))
test_cmds = ['\n'.join(test_cmds)]
for (ind, test) in enumerate(test_cmds):
job = self.start_job(test, cluster_queue, verbose)
job.wait()
# Analyse tests as they finish.
if cluster_queue:
# Did all of them at once.
for (test_input, test_arg) in self.inputs_args:
self.verify_job(test_input, test_arg, verbose, rundir)
else:
# Did one job at a time.
(test_input, test_arg) = self.inputs_args[ind]
err = []
if self.output:
try:
self.move_output_to_test_output(test_files[ind])
except exceptions.RunError:
err.append(sys.exc_info()[1])
status = validation.Status()
if job.returncode != 0:
err.insert(
0, 'Error running job. Return code: %i' %
job.returncode)
(status, msg) = self.skip_job(test_input, test_arg,
verbose)
if status.skipped():
self._update_status(status, (test_input, test_arg))
if verbose > 0 and verbose < 3:
sys.stdout.write(
util.info_line(self.path, test_input, test_arg,
rundir))
status.print_status(msg, verbose)
elif err:
# re-raise first error we hit.
raise exceptions.RunError(err[0])
else:
self.verify_job(test_input, test_arg, verbose, rundir)
except exceptions.RunError:
err = sys.exc_info()[1]
if verbose > 2:
err = 'Test(s) in %s failed.\n%s' % (self.path, err)
status = validation.Status([False])
self._update_status(status, (test_input, test_arg))
if verbose > 0 and verbose < 3:
info_line = util.info_line(self.path, test_input, test_arg,
rundir)
sys.stdout.write(info_line)
status.print_status(err, verbose)
# Shouldn't run remaining tests after such a catastrophic failure.
# Mark all remaining tests as skipped so the user knows that they
# weren't run.
err = 'Previous test in %s caused a system failure.' % (self.path)
status = validation.Status(name='skipped')
for ((test_input, test_arg), stat) in self.status.items():
if not self.status[(test_input, test_arg)]:
self._update_status(status, (test_input, test_arg))
if verbose > 2:
cmd = self.test_program.run_cmd(
test_input, test_arg, self.nprocs)
print('Test using %s in %s' % (cmd, self.path))
elif verbose > 0:
info_line = util.info_line(self.path, test_input,
test_arg, rundir)
sys.stdout.write(info_line)
status.print_status(err, verbose)
def extract_data(self, input_file, args, verbose=1):
'''Extract data from output file.
Assume function is executed in self.path.'''
tp_ptr = self.test_program
if tp_ptr.data_tag:
# Using internal data extraction function.
data_files = [
tp_ptr.select_benchmark_file(self.path, input_file, args),
util.testcode_filename(FILESTEM['test'],
tp_ptr.test_id, input_file, args),
]
if verbose > 2:
print('Analysing output using data_tag %s in %s on files %s.' %
(tp_ptr.data_tag, self.path, ' and '.join(data_files)))
outputs = [util.extract_tagged_data(tp_ptr.data_tag, dfile)
for dfile in data_files]
else:
# Using external data extraction script.
# Get extraction commands.
extract_cmds = tp_ptr.extract_cmd(self.path, input_file, args)
# Extract data.
outputs = []
for cmd in extract_cmds:
try:
if verbose > 2:
print('Analysing output using %s in %s.' %
(cmd, self.path))
# Samuel Ponce: Popen.wait() creates deadlock if the data is too large
# See documented issue for example in:
# https://docs.python.org/2/library/subprocess.html#subprocess.Popen.returncode
#
# Previous code that create deadlock:
#extract_popen = subprocess.Popen(cmd, shell=True,
# stdout=subprocess.PIPE, stderr=subprocess.PIPE)
#extract_popen.wait()
#
# New code (this might not be the best but work for me):
extract_popen = subprocess.Popen(cmd, bufsize=1, shell=True,
stdin=open(os.devnull), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
lines = []
for line in iter(extract_popen.stdout.readline, ''):
#print line,
lines.append(line)
except OSError:
# slightly odd syntax in order to be compatible with python
# 2.5 and python 2.6/3
err = 'Analysing output failed: %s' % (sys.exc_info()[1],)
raise exceptions.AnalysisError(err)
# Convert data string from extract command to dictionary format.
# SP: Because of the above change, the test below cannot be done:
#if extract_popen.returncode != 0:
# err = extract_popen.communicate()[1].decode('utf-8')
# err = 'Analysing output failed: %s' % (err)
# raise exceptions.AnalysisError(err)
#data_string = extract_popen.communicate()[0].decode('utf-8')
data_string = ''.join(lines)
if self.test_program.extract_fmt == 'table':
outputs.append(util.dict_table_string(data_string))
elif self.test_program.extract_fmt == 'yaml':
outputs.append({})
# convert values to be in a tuple so the format matches
# that from dict_table_string.
# ensure all keys are strings so they can be sorted
# (different data types cause problems!)
for (key, val) in yaml.safe_load(data_string).items():
if isinstance(val, list):
outputs[-1][str(key)] = tuple(val)
else:
outputs[-1][str(key)] = tuple((val,))
return tuple(outputs)
