def extract_data(self, input_file, args, verbose=1):
'''Extract data from output file.
Assume function is executed in self.path.'''
tp_ptr = self.test_program
if tp_ptr.data_tag:
# Using internal data extraction function.
data_files = [
util.testcode_filename(FILESTEM['benchmark'],
tp_ptr.benchmark, input_file, args),
util.testcode_filename(FILESTEM['test'],
tp_ptr.test_id, input_file, args),
]
if verbose > 2:
print('Analysing output using data_tag %s in %s on files %s.' %
(tp_ptr.data_tag, self.path, ' and '.join(data_files)))
outputs = [util.extract_tagged_data(tp_ptr.data_tag, dfile)
for dfile in data_files]
else:
# Using external data extraction script.
# Get extraction commands.
extract_cmds = self.test_program.extract_cmd(input_file, args)
# Extract data.
outputs = []
for cmd in extract_cmds:
try:
if verbose > 2:
print('Analysing output using %s in %s.' %
(cmd, self.path))
extract_popen = subprocess.Popen(cmd, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
extract_popen.wait()
except OSError:
# slightly odd syntax in order to be compatible with python
# 2.5 and python 2.6/3
err = 'Analysing output failed: %s' % (sys.exc_info()[1],)
raise exceptions.AnalysisError(err)
# Convert data string from extract command to dictionary format.
if extract_popen.returncode != 0:
err = extract_popen.communicate()[1].decode('utf-8')
err = 'Analysing output failed: %s' % (err)
raise exceptions.AnalysisError(err)
data_string = extract_popen.communicate()[0].decode('utf-8')
if self.test_program.extract_fmt == 'table':
outputs.append(util.dict_table_string(data_string))
elif self.test_program.extract_fmt == 'yaml':
outputs.append({})
# convert values to be in a tuple so the format matches
# that from dict_table_string.
# ensure all keys are strings so they can be sorted
# (different data types cause problems!)
for (key, val) in yaml.safe_load(data_string).items():
if isinstance(val, list):
outputs[-1][str(key)] = tuple(val)
else:
outputs[-1][str(key)] = tuple((val,))
return tuple(outputs)
def extract_data(self, input_file, args, verbose=1):
'''Extract data from output file.
Assume function is executed in self.path.'''
tp_ptr = self.test_program
if tp_ptr.data_tag:
# Using internal data extraction function.
data_files = [
tp_ptr.select_benchmark_file(self.path, input_file, args),
util.testcode_filename(FILESTEM['test'],
tp_ptr.test_id, input_file, args),
]
if verbose > 2:
print(('Analysing output using data_tag %s in %s on files %s.' %
(tp_ptr.data_tag, self.path, ' and '.join(data_files))))
outputs = [util.extract_tagged_data(tp_ptr.data_tag, dfile)
for dfile in data_files]
else:
# Using external data extraction script.
# Get extraction commands.
extract_cmds = tp_ptr.extract_cmd(self.path, input_file, args)
# Extract data.
outputs = []
for cmd in extract_cmds:
try:
if verbose > 2:
print(('Analysing output using %s in %s.' %
(cmd, self.path)))
extract_popen = subprocess.run(cmd, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except OSError:
# slightly odd syntax in order to be compatible with python
# 2.5 and python 2.6/3
err = 'Analysing output failed: %s' % (sys.exc_info()[1],)
raise exceptions.AnalysisError(err)
# Convert data string from extract command to dictionary format.
if extract_popen.returncode != 0:
err = extract_popen.communicate()[1].decode('utf-8')
err = 'Analysing output failed: %s' % (err)
raise exceptions.AnalysisError(err)
data_string = extract_popen.stdout.decode('utf-8')
if self.test_program.extract_fmt == 'table':
outputs.append(util.dict_table_string(data_string))
elif self.test_program.extract_fmt == 'yaml':
outputs.append({})
# convert values to be in a tuple so the format matches
# that from dict_table_string.
# ensure all keys are strings so they can be sorted
# (different data types cause problems!)
for (key, val) in list(yaml.safe_load(data_string).items()):
if isinstance(val, list):
outputs[-1][str(key)] = tuple(val)
else:
outputs[-1][str(key)] = tuple((val,))
return tuple(outputs)
def extract_data(self, input_file, args, verbose=1):
'''Extract data from output file.
Assume function is executed in self.path.'''
tp_ptr = self.test_program
if tp_ptr.data_tag:
# Using internal data extraction function.
data_files = [
tp_ptr.select_benchmark_file(self.path, input_file, args),
util.testcode_filename(FILESTEM['test'], tp_ptr.test_id,
input_file, args),
]
if verbose > 2:
print('Analysing output using data_tag %s in %s on files %s.' %
(tp_ptr.data_tag, self.path, ' and '.join(data_files)))
outputs = [
util.extract_tagged_data(tp_ptr.data_tag, dfile)
for dfile in data_files
]
else:
# Using external data extraction script.
# Get extraction commands.
extract_cmds = tp_ptr.extract_cmd(self.path, input_file, args)
# Extract data.
outputs = []
for cmd in extract_cmds:
try:
if verbose > 2:
print('Analysing output using %s in %s.' %
(cmd, self.path))
# Samuel Ponce: Popen.wait() creates deadlock if the data is too large
# See documented issue for example in:
# https://docs.python.org/2/library/subprocess.html#subprocess.Popen.returncode
#
# Previous code that create deadlock:
#extract_popen = subprocess.Popen(cmd, shell=True,
# stdout=subprocess.PIPE, stderr=subprocess.PIPE)
#extract_popen.wait()
#
# New code (this might not be the best but work for me):
extract_popen = subprocess.Popen(cmd,
bufsize=1,
shell=True,
stdin=open(os.devnull),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
lines = []
for line in iter(extract_popen.stdout.readline, ''):
#print line,
lines.append(line)
except OSError:
# slightly odd syntax in order to be compatible with python
# 2.5 and python 2.6/3
err = 'Analysing output failed: %s' % (sys.exc_info()[1], )
raise exceptions.AnalysisError(err)
# Convert data string from extract command to dictionary format.
# SP: Because of the above change, the test below cannot be done:
#if extract_popen.returncode != 0:
# err = extract_popen.communicate()[1].decode('utf-8')
# err = 'Analysing output failed: %s' % (err)
# raise exceptions.AnalysisError(err)
#data_string = extract_popen.communicate()[0].decode('utf-8')
data_string = ''.join(lines)
if self.test_program.extract_fmt == 'table':
outputs.append(util.dict_table_string(data_string))
elif self.test_program.extract_fmt == 'yaml':
outputs.append({})
# convert values to be in a tuple so the format matches
# that from dict_table_string.
# ensure all keys are strings so they can be sorted
# (different data types cause problems!)
for (key, val) in yaml.safe_load(data_string).items():
if isinstance(val, list):
outputs[-1][str(key)] = tuple(val)
else:
outputs[-1][str(key)] = tuple((val, ))
return tuple(outputs)
def extract_data(self, input_file, args, verbose=1):
'''Extract data from output file.
Assume function is executed in self.path.'''
tp_ptr = self.test_program
if tp_ptr.data_tag:
# Using internal data extraction function.
data_files = [
tp_ptr.select_benchmark_file(self.path, input_file, args),
util.testcode_filename(FILESTEM['test'],
tp_ptr.test_id, input_file, args),
]
if verbose > 2:
print('Analysing output using data_tag %s in %s on files %s.' %
(tp_ptr.data_tag, self.path, ' and '.join(data_files)))
outputs = [util.extract_tagged_data(tp_ptr.data_tag, dfile)
for dfile in data_files]
else:
# Using external data extraction script.
# Get extraction commands.
extract_cmds = tp_ptr.extract_cmd(self.path, input_file, args)
# Extract data.
outputs = []
for cmd in extract_cmds:
try:
if verbose > 2:
print('Analysing output using %s in %s.' %
(cmd, self.path))
# Samuel Ponce: Popen.wait() creates deadlock if the data is too large
# See documented issue for example in:
# https://docs.python.org/2/library/subprocess.html#subprocess.Popen.returncode
#
# Previous code that create deadlock:
#extract_popen = subprocess.Popen(cmd, shell=True,
# stdout=subprocess.PIPE, stderr=subprocess.PIPE)
#extract_popen.wait()
#
# New code (this might not be the best but work for me):
extract_popen = subprocess.Popen(cmd, bufsize=1, shell=True,
stdin=open(os.devnull), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
lines = []
for line in iter(extract_popen.stdout.readline, ''):
#print line,
lines.append(line)
except OSError:
# slightly odd syntax in order to be compatible with python
# 2.5 and python 2.6/3
err = 'Analysing output failed: %s' % (sys.exc_info()[1],)
raise exceptions.AnalysisError(err)
# Convert data string from extract command to dictionary format.
# SP: Because of the above change, the test below cannot be done:
#if extract_popen.returncode != 0:
# err = extract_popen.communicate()[1].decode('utf-8')
# err = 'Analysing output failed: %s' % (err)
# raise exceptions.AnalysisError(err)
#data_string = extract_popen.communicate()[0].decode('utf-8')
data_string = ''.join(lines)
if self.test_program.extract_fmt == 'table':
outputs.append(util.dict_table_string(data_string))
elif self.test_program.extract_fmt == 'yaml':
outputs.append({})
# convert values to be in a tuple so the format matches
# that from dict_table_string.
# ensure all keys are strings so they can be sorted
# (different data types cause problems!)
for (key, val) in yaml.safe_load(data_string).items():
if isinstance(val, list):
outputs[-1][str(key)] = tuple(val)
else:
outputs[-1][str(key)] = tuple((val,))
return tuple(outputs)