def compare_data(benchmark, test, default_tolerance, tolerances,
ignore_fields=None):
'''Compare two data dictionaries.'''
ignored_params = compat.compat_set(ignore_fields or tuple())
bench_params = compat.compat_set(benchmark) - ignored_params
test_params = compat.compat_set(test) - ignored_params
# Check both the key names and the number of keys in case there are
# different numbers of duplicate keys.
comparable = (bench_params == test_params)
key_counts = dict((key,0) for key in bench_params | test_params)
for (key, val) in benchmark.items():
if key not in ignored_params:
key_counts[key] += len(val)
for (key, val) in test.items():
if key not in ignored_params:
key_counts[key] -= len(val)
comparable = comparable and compat.compat_all(kc == 0 for kc in key_counts.values())
status = Status()
msg = []
if not comparable:
status = Status([False])
bench_only = bench_params - test_params
test_only = test_params - bench_params
msg.append('Different sets of data extracted from benchmark and test.')
if bench_only:
msg.append(" Data only in benchmark: %s." % ", ".join(bench_only))
if test_only:
msg.append(" Data only in test: %s." % ", ".join(test_only))
bench_more = [key for key in key_counts
if key_counts[key] > 0 and key not in bench_only]
test_more = [key for key in key_counts
if key_counts[key] < 0 and key not in test_only]
if bench_more:
msg.append(" More data in benchmark than in test: %s." %
", ".join(bench_more))
if test_more:
msg.append(" More data in test than in benchmark: %s." %
", ".join(test_more))
for param in (bench_params & test_params):
param_tol = tolerances.get(param, default_tolerance)
if param_tol == default_tolerance:
# See if there's a regex that matches.
tol_matches = [tol for tol in tolerances.values()
if tol.name and re.match(tol.name, param)]
if tol_matches:
param_tol = tol_matches[0]
if len(tol_matches) > 1:
warnings.warn('Multiple tolerance regexes match. '
'Using %s.' % (param_tol.name))
for bench_value, test_value in zip(benchmark[param], test[param]):
key_status, err = param_tol.validate(test_value, bench_value, param)
status += key_status
if not key_status.passed() and err:
msg.append(err)
return (comparable, status, "\n".join(msg))
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
else:
# Compare values we care about...
cmp_vals = ['test_program', 'path', 'inputs_args', 'output',
'nprocs', 'min_nprocs', 'max_nprocs', 'submit_template',
'default_tolerance', 'tolerances', 'status']
comparison = tuple(getattr(other, cmp_val) == getattr(self, cmp_val) for cmp_val in cmp_vals)
return compat.compat_all(comparison)
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
else:
# Compare values we care about...
cmp_vals = ['test_program', 'path', 'inputs_args', 'output',
'nprocs', 'min_nprocs', 'max_nprocs', 'submit_template',
'default_tolerance', 'tolerances', 'status']
comparison = tuple(getattr(other, cmp_val) == getattr(self, cmp_val) for cmp_val in cmp_vals)
return compat.compat_all(comparison)
def dict_table_string(table_string):
'''Read a data table from a string into a dictionary.
The first row and any subsequent rows containing no numbers are assumed to form
headers of a subtable, and so form the keys for the subsequent subtable.
Values, where possible, are converted to floats.
e.g. a b c a -> {'a':(1,4,7,8), 'b':(2,5), 'c':(3,6)}
1 2 3 7
4 5 6 8
and
a b c -> {'a':(1,4,7), 'b':(2,5,8), 'c':(3,6), 'd':(9), 'e':(6)}
1 2 3
4 5 6
a b d e
7 8 9 6
'''
data = [i.split() for i in table_string.splitlines()]
# Convert to numbers where appropriate
data = [[try_floatify(val) for val in dline] for dline in data]
data_dict = {}
head = []
for dline in data:
# Test if all items are strings; if so start a new subtable.
# We actually test if all items are not floats, as python 3 can return
# a bytes variable from subprocess whereas (e.g.) python 2.4 returns a
# str. Testing for this is problematic as the bytes type does not
# exist in python 2.4. Fortunately we have converted all items to
# floats if possible, so can just test for the inverse condition...
if compat.compat_all(type(val) is not float for val in dline):
# header of new subtable
head = dline
for val in head:
if val not in data_dict:
data_dict[val] = []
else:
if len(dline) > len(head):
err = 'Table missing column heading(s):\n%s' % (table_string)
raise exceptions.AnalysisError(err)
for (ind, val) in enumerate(dline):
# Add data to appropriate key.
# Note that this handles the case where the same column heading
# occurs multiple times in the same subtable and does not
# overwrite the previous column with the same heading.
data_dict[head[ind]].append(val)
# We shouldn't change the data from this point: convert entries to tuples.
for (key, val) in data_dict.items():
data_dict[key] = tuple(val)
return data_dict
def dict_table_string(table_string):
'''Read a data table from a string into a dictionary.
The first row and any subsequent rows containing no numbers are assumed to form
headers of a subtable, and so form the keys for the subsequent subtable.
Values, where possible, are converted to floats.
e.g. a b c a -> {'a':(1,4,7,8), 'b':(2,5), 'c':(3,6)}
1 2 3 7
4 5 6 8
and
a b c -> {'a':(1,4,7), 'b':(2,5,8), 'c':(3,6), 'd':(9), 'e':(6)}
1 2 3
4 5 6
a b d e
7 8 9 6
'''
data = [i.split() for i in table_string.splitlines()]
# Convert to numbers where appropriate
data = [[try_floatify(val) for val in dline] for dline in data]
data_dict = {}
head = []
for dline in data:
# Test if all items are strings; if so start a new subtable.
# We actually test if all items are not floats, as python 3 can return
# a bytes variable from subprocess whereas (e.g.) python 2.4 returns a
# str. Testing for this is problematic as the bytes type does not
# exist in python 2.4. Fortunately we have converted all items to
# floats if possible, so can just test for the inverse condition...
if compat.compat_all(type(val) is not float for val in dline):
# header of new subtable
head = dline
for val in head:
if val not in data_dict:
data_dict[val] = []
else:
if len(dline) > len(head):
err = 'Table missing column heading(s):\n%s' % (table_string)
raise exceptions.AnalysisError(err)
for (ind, val) in enumerate(dline):
# Add data to appropriate key.
# Note that this handles the case where the same column heading
# occurs multiple times in the same subtable and does not
# overwrite the previous column with the same heading.
data_dict[head[ind]].append(val)
# We shouldn't change the data from this point: convert entries to tuples.
for (key, val) in data_dict.items():
data_dict[key] = tuple(val)
return data_dict
def __init__(self, bools=None, status=None, name=None):
(self._unknown, self._skipped) = (-2, -1)
(self._passed, self._partial, self._failed) = (0, 1, 2)
if name is not None:
setattr(self, 'status', getattr(self, '_' + name))
elif status is not None:
self.status = status
elif bools:
if compat.compat_all(bools):
self.status = self._passed
elif compat.compat_any(bools):
self.status = self._partial
else:
self.status = self._failed
else:
self.status = self._unknown
def __init__(self, bools=None, status=None, name=None):
(self._unknown, self._skipped) = (-2, -1)
(self._passed, self._partial, self._failed) = (0, 1, 2)
if name is not None:
setattr(self, 'status', getattr(self, '_'+name))
elif status is not None:
self.status = status
elif bools:
if compat.compat_all(bools):
self.status = self._passed
elif compat.compat_any(bools):
self.status = self._partial
else:
self.status = self._failed
else:
self.status = self._unknown
def compare_data(benchmark,
test,
default_tolerance,
tolerances,
ignore_fields=None):
'''Compare two data dictionaries.'''
ignored_params = compat.compat_set(ignore_fields or tuple())
bench_params = compat.compat_set(benchmark) - ignored_params
test_params = compat.compat_set(test) - ignored_params
# Check both the key names and the number of keys in case there are
# different numbers of duplicate keys.
comparable = (bench_params == test_params)
key_counts = dict((key, 0) for key in bench_params | test_params)
for (key, val) in benchmark.items():
if key not in ignored_params:
key_counts[key] += len(val)
for (key, val) in test.items():
if key not in ignored_params:
key_counts[key] -= len(val)
comparable = comparable and compat.compat_all(
kc == 0 for kc in key_counts.values())
status = Status()
msg = []
if not comparable:
status = Status([False])
bench_only = bench_params - test_params
test_only = test_params - bench_params
msg.append('Different sets of data extracted from benchmark and test.')
if bench_only:
msg.append(" Data only in benchmark: %s." %
", ".join(bench_only))
if test_only:
msg.append(" Data only in test: %s." % ", ".join(test_only))
bench_more = [
key for key in key_counts
if key_counts[key] > 0 and key not in bench_only
]
test_more = [
key for key in key_counts
if key_counts[key] < 0 and key not in test_only
]
if bench_more:
msg.append(" More data in benchmark than in test: %s." %
", ".join(bench_more))
if test_more:
msg.append(" More data in test than in benchmark: %s." %
", ".join(test_more))
for param in (bench_params & test_params):
param_tol = tolerances.get(param, default_tolerance)
if param_tol == default_tolerance:
# See if there's a regex that matches.
tol_matches = [
tol for tol in tolerances.values()
if tol.name and re.match(tol.name, param)
]
if tol_matches:
param_tol = tol_matches[0]
if len(tol_matches) > 1:
warnings.warn('Multiple tolerance regexes match. '
'Using %s.' % (param_tol.name))
for bench_value, test_value in zip(benchmark[param], test[param]):
key_status, err = param_tol.validate(test_value, bench_value,
param)
status += key_status
if not key_status.passed() and err:
msg.append(err)
return (comparable, status, "\n".join(msg))
