def compare_data(benchmark, test, default_tolerance, tolerances, ignore_fields=None): '''Compare two data dictionaries.''' ignored_params = compat.compat_set(ignore_fields or tuple()) bench_params = compat.compat_set(benchmark) - ignored_params test_params = compat.compat_set(test) - ignored_params # Check both the key names and the number of keys in case there are # different numbers of duplicate keys. comparable = (bench_params == test_params) key_counts = dict((key,0) for key in bench_params | test_params) for (key, val) in benchmark.items(): if key not in ignored_params: key_counts[key] += len(val) for (key, val) in test.items(): if key not in ignored_params: key_counts[key] -= len(val) comparable = comparable and compat.compat_all(kc == 0 for kc in key_counts.values()) status = Status() msg = [] if not comparable: status = Status([False]) bench_only = bench_params - test_params test_only = test_params - bench_params msg.append('Different sets of data extracted from benchmark and test.') if bench_only: msg.append(" Data only in benchmark: %s." % ", ".join(bench_only)) if test_only: msg.append(" Data only in test: %s." % ", ".join(test_only)) bench_more = [key for key in key_counts if key_counts[key] > 0 and key not in bench_only] test_more = [key for key in key_counts if key_counts[key] < 0 and key not in test_only] if bench_more: msg.append(" More data in benchmark than in test: %s." % ", ".join(bench_more)) if test_more: msg.append(" More data in test than in benchmark: %s." % ", ".join(test_more)) for param in (bench_params & test_params): param_tol = tolerances.get(param, default_tolerance) if param_tol == default_tolerance: # See if there's a regex that matches. tol_matches = [tol for tol in tolerances.values() if tol.name and re.match(tol.name, param)] if tol_matches: param_tol = tol_matches[0] if len(tol_matches) > 1: warnings.warn('Multiple tolerance regexes match. ' 'Using %s.' % (param_tol.name)) for bench_value, test_value in zip(benchmark[param], test[param]): key_status, err = param_tol.validate(test_value, bench_value, param) status += key_status if not key_status.passed() and err: msg.append(err) return (comparable, status, "\n".join(msg))
def __eq__(self, other): if not isinstance(other, self.__class__): return False else: # Compare values we care about... cmp_vals = ['test_program', 'path', 'inputs_args', 'output', 'nprocs', 'min_nprocs', 'max_nprocs', 'submit_template', 'default_tolerance', 'tolerances', 'status'] comparison = tuple(getattr(other, cmp_val) == getattr(self, cmp_val) for cmp_val in cmp_vals) return compat.compat_all(comparison)
def __eq__(self, other): if not isinstance(other, self.__class__): return False else: # Compare values we care about... cmp_vals = ['test_program', 'path', 'inputs_args', 'output', 'nprocs', 'min_nprocs', 'max_nprocs', 'submit_template', 'default_tolerance', 'tolerances', 'status'] comparison = tuple(getattr(other, cmp_val) == getattr(self, cmp_val) for cmp_val in cmp_vals) return compat.compat_all(comparison)
def dict_table_string(table_string): '''Read a data table from a string into a dictionary. The first row and any subsequent rows containing no numbers are assumed to form headers of a subtable, and so form the keys for the subsequent subtable. Values, where possible, are converted to floats. e.g. a b c a -> {'a':(1,4,7,8), 'b':(2,5), 'c':(3,6)} 1 2 3 7 4 5 6 8 and a b c -> {'a':(1,4,7), 'b':(2,5,8), 'c':(3,6), 'd':(9), 'e':(6)} 1 2 3 4 5 6 a b d e 7 8 9 6 ''' data = [i.split() for i in table_string.splitlines()] # Convert to numbers where appropriate data = [[try_floatify(val) for val in dline] for dline in data] data_dict = {} head = [] for dline in data: # Test if all items are strings; if so start a new subtable. # We actually test if all items are not floats, as python 3 can return # a bytes variable from subprocess whereas (e.g.) python 2.4 returns a # str. Testing for this is problematic as the bytes type does not # exist in python 2.4. Fortunately we have converted all items to # floats if possible, so can just test for the inverse condition... if compat.compat_all(type(val) is not float for val in dline): # header of new subtable head = dline for val in head: if val not in data_dict: data_dict[val] = [] else: if len(dline) > len(head): err = 'Table missing column heading(s):\n%s' % (table_string) raise exceptions.AnalysisError(err) for (ind, val) in enumerate(dline): # Add data to appropriate key. # Note that this handles the case where the same column heading # occurs multiple times in the same subtable and does not # overwrite the previous column with the same heading. data_dict[head[ind]].append(val) # We shouldn't change the data from this point: convert entries to tuples. for (key, val) in data_dict.items(): data_dict[key] = tuple(val) return data_dict
def dict_table_string(table_string): '''Read a data table from a string into a dictionary. The first row and any subsequent rows containing no numbers are assumed to form headers of a subtable, and so form the keys for the subsequent subtable. Values, where possible, are converted to floats. e.g. a b c a -> {'a':(1,4,7,8), 'b':(2,5), 'c':(3,6)} 1 2 3 7 4 5 6 8 and a b c -> {'a':(1,4,7), 'b':(2,5,8), 'c':(3,6), 'd':(9), 'e':(6)} 1 2 3 4 5 6 a b d e 7 8 9 6 ''' data = [i.split() for i in table_string.splitlines()] # Convert to numbers where appropriate data = [[try_floatify(val) for val in dline] for dline in data] data_dict = {} head = [] for dline in data: # Test if all items are strings; if so start a new subtable. # We actually test if all items are not floats, as python 3 can return # a bytes variable from subprocess whereas (e.g.) python 2.4 returns a # str. Testing for this is problematic as the bytes type does not # exist in python 2.4. Fortunately we have converted all items to # floats if possible, so can just test for the inverse condition... if compat.compat_all(type(val) is not float for val in dline): # header of new subtable head = dline for val in head: if val not in data_dict: data_dict[val] = [] else: if len(dline) > len(head): err = 'Table missing column heading(s):\n%s' % (table_string) raise exceptions.AnalysisError(err) for (ind, val) in enumerate(dline): # Add data to appropriate key. # Note that this handles the case where the same column heading # occurs multiple times in the same subtable and does not # overwrite the previous column with the same heading. data_dict[head[ind]].append(val) # We shouldn't change the data from this point: convert entries to tuples. for (key, val) in data_dict.items(): data_dict[key] = tuple(val) return data_dict
def __init__(self, bools=None, status=None, name=None): (self._unknown, self._skipped) = (-2, -1) (self._passed, self._partial, self._failed) = (0, 1, 2) if name is not None: setattr(self, 'status', getattr(self, '_' + name)) elif status is not None: self.status = status elif bools: if compat.compat_all(bools): self.status = self._passed elif compat.compat_any(bools): self.status = self._partial else: self.status = self._failed else: self.status = self._unknown
def __init__(self, bools=None, status=None, name=None): (self._unknown, self._skipped) = (-2, -1) (self._passed, self._partial, self._failed) = (0, 1, 2) if name is not None: setattr(self, 'status', getattr(self, '_'+name)) elif status is not None: self.status = status elif bools: if compat.compat_all(bools): self.status = self._passed elif compat.compat_any(bools): self.status = self._partial else: self.status = self._failed else: self.status = self._unknown
def compare_data(benchmark, test, default_tolerance, tolerances, ignore_fields=None): '''Compare two data dictionaries.''' ignored_params = compat.compat_set(ignore_fields or tuple()) bench_params = compat.compat_set(benchmark) - ignored_params test_params = compat.compat_set(test) - ignored_params # Check both the key names and the number of keys in case there are # different numbers of duplicate keys. comparable = (bench_params == test_params) key_counts = dict((key, 0) for key in bench_params | test_params) for (key, val) in benchmark.items(): if key not in ignored_params: key_counts[key] += len(val) for (key, val) in test.items(): if key not in ignored_params: key_counts[key] -= len(val) comparable = comparable and compat.compat_all( kc == 0 for kc in key_counts.values()) status = Status() msg = [] if not comparable: status = Status([False]) bench_only = bench_params - test_params test_only = test_params - bench_params msg.append('Different sets of data extracted from benchmark and test.') if bench_only: msg.append(" Data only in benchmark: %s." % ", ".join(bench_only)) if test_only: msg.append(" Data only in test: %s." % ", ".join(test_only)) bench_more = [ key for key in key_counts if key_counts[key] > 0 and key not in bench_only ] test_more = [ key for key in key_counts if key_counts[key] < 0 and key not in test_only ] if bench_more: msg.append(" More data in benchmark than in test: %s." % ", ".join(bench_more)) if test_more: msg.append(" More data in test than in benchmark: %s." % ", ".join(test_more)) for param in (bench_params & test_params): param_tol = tolerances.get(param, default_tolerance) if param_tol == default_tolerance: # See if there's a regex that matches. tol_matches = [ tol for tol in tolerances.values() if tol.name and re.match(tol.name, param) ] if tol_matches: param_tol = tol_matches[0] if len(tol_matches) > 1: warnings.warn('Multiple tolerance regexes match. ' 'Using %s.' % (param_tol.name)) for bench_value, test_value in zip(benchmark[param], test[param]): key_status, err = param_tol.validate(test_value, bench_value, param) status += key_status if not key_status.passed() and err: msg.append(err) return (comparable, status, "\n".join(msg))