def compare_data(benchmark, test, default_tolerance, tolerances, ignore_fields=None): '''Compare two data dictionaries.''' ignored_params = compat.compat_set(ignore_fields or tuple()) bench_params = compat.compat_set(benchmark) - ignored_params test_params = compat.compat_set(test) - ignored_params # Check both the key names and the number of keys in case there are # different numbers of duplicate keys. comparable = (bench_params == test_params) key_counts = dict((key,0) for key in bench_params | test_params) for (key, val) in benchmark.items(): if key not in ignored_params: key_counts[key] += len(val) for (key, val) in test.items(): if key not in ignored_params: key_counts[key] -= len(val) comparable = comparable and compat.compat_all(kc == 0 for kc in key_counts.values()) status = Status() msg = [] if not comparable: status = Status([False]) bench_only = bench_params - test_params test_only = test_params - bench_params msg.append('Different sets of data extracted from benchmark and test.') if bench_only: msg.append(" Data only in benchmark: %s." % ", ".join(bench_only)) if test_only: msg.append(" Data only in test: %s." % ", ".join(test_only)) bench_more = [key for key in key_counts if key_counts[key] > 0 and key not in bench_only] test_more = [key for key in key_counts if key_counts[key] < 0 and key not in test_only] if bench_more: msg.append(" More data in benchmark than in test: %s." % ", ".join(bench_more)) if test_more: msg.append(" More data in test than in benchmark: %s." % ", ".join(test_more)) for param in (bench_params & test_params): param_tol = tolerances.get(param, default_tolerance) if param_tol == default_tolerance: # See if there's a regex that matches. tol_matches = [tol for tol in tolerances.values() if tol.name and re.match(tol.name, param)] if tol_matches: param_tol = tol_matches[0] if len(tol_matches) > 1: warnings.warn('Multiple tolerance regexes match. ' 'Using %s.' % (param_tol.name)) for bench_value, test_value in zip(benchmark[param], test[param]): key_status, err = param_tol.validate(test_value, bench_value, param) status += key_status if not key_status.passed() and err: msg.append(err) return (comparable, status, "\n".join(msg))
def select_tests(all_tests, test_categories, selected_categories, prefix=''): '''Return the set of tests contained by the selected test categories.''' test_categories['_all_'] = [test.path for test in all_tests] if ('_default_' in selected_categories and '_default_' not in test_categories): selected_categories = ['_all_'] # Recursively expand job categories. while compat.compat_any(cat in test_categories for cat in selected_categories): tmp = [] for cat in selected_categories: if cat in test_categories: tmp.extend(test_categories[cat]) else: # cat has been fully expanded and now refers to a test # contained within the directory named cat. tmp.append(cat) selected_categories = tmp # Select tests to run. tests = [] parent = lambda pdir, cdir: \ not os.path.relpath(cdir, start=pdir).startswith(os.pardir) for cat in compat.compat_set(selected_categories): # test paths are relative to the config directory but absolute paths # are stored . found = False cat_paths = glob.glob(os.path.join(prefix, cat)) for test in all_tests: if cat == test.name: found = True tests.append(test) elif compat.compat_any( os.path.exists(path) and os.path.samefile(path, test.path) for path in cat_paths): found = True tests.append(test) elif compat.compat_any( parent(path, test.path) for path in cat_paths): # test contained within a subdirectory of a cat_path. found = True tests.append(test) if not found: print('WARNING: %s test/category not found.\n' % cat) # Only want to run each test once. tests = list(compat.compat_set(tests)) return tests
def select_tests(all_tests, test_categories, selected_categories, prefix=''): '''Return the set of tests contained by the selected test categories.''' test_categories['_all_'] = [test.path for test in all_tests] if ('_default_' in selected_categories and '_default_' not in test_categories): selected_categories = ['_all_'] # Recursively expand job categories. while compat.compat_any( cat in test_categories for cat in selected_categories ): tmp = [] for cat in selected_categories: if cat in test_categories: tmp.extend(test_categories[cat]) else: # cat has been fully expanded and now refers to a test # contained within the directory named cat. tmp.append(cat) selected_categories = tmp # Select tests to run. tests = [] parent = lambda pdir, cdir: \ not os.path.relpath(cdir, start=pdir).startswith(os.pardir) for cat in compat.compat_set(selected_categories): # test paths are relative to the config directory but absolute paths # are stored . found = False cat_paths = glob.glob(os.path.join(prefix, cat)) for test in all_tests: if cat == test.name: found = True tests.append(test) elif compat.compat_any(os.path.exists(path) and os.path.samefile(path, test.path) for path in cat_paths): found = True tests.append(test) elif compat.compat_any(parent(path, test.path) for path in cat_paths): # test contained within a subdirectory of a cat_path. found = True tests.append(test) if not found: print('WARNING: %s test/category not found.\n' % cat) # Only want to run each test once. tests = list(compat.compat_set(tests)) return tests
def compare_data(benchmark, test, default_tolerance, tolerances, ignore_fields=None): '''Compare two data dictionaries.''' ignored_params = compat.compat_set(ignore_fields or tuple()) bench_params = compat.compat_set(benchmark) - ignored_params test_params = compat.compat_set(test) - ignored_params # Check both the key names and the number of keys in case there are # different numbers of duplicate keys. comparable = (bench_params == test_params) key_counts = dict((key, 0) for key in bench_params | test_params) for (key, val) in benchmark.items(): if key not in ignored_params: key_counts[key] += len(val) for (key, val) in test.items(): if key not in ignored_params: key_counts[key] -= len(val) comparable = comparable and compat.compat_all( kc == 0 for kc in key_counts.values()) status = Status() msg = [] if not comparable: status = Status([False]) bench_only = bench_params - test_params test_only = test_params - bench_params msg.append('Different sets of data extracted from benchmark and test.') if bench_only: msg.append(" Data only in benchmark: %s." % ", ".join(bench_only)) if test_only: msg.append(" Data only in test: %s." % ", ".join(test_only)) bench_more = [ key for key in key_counts if key_counts[key] > 0 and key not in bench_only ] test_more = [ key for key in key_counts if key_counts[key] < 0 and key not in test_only ] if bench_more: msg.append(" More data in benchmark than in test: %s." % ", ".join(bench_more)) if test_more: msg.append(" More data in test than in benchmark: %s." % ", ".join(test_more)) for param in (bench_params & test_params): param_tol = tolerances.get(param, default_tolerance) if param_tol == default_tolerance: # See if there's a regex that matches. tol_matches = [ tol for tol in tolerances.values() if tol.name and re.match(tol.name, param) ] if tol_matches: param_tol = tol_matches[0] if len(tol_matches) > 1: warnings.warn('Multiple tolerance regexes match. ' 'Using %s.' % (param_tol.name)) for bench_value, test_value in zip(benchmark[param], test[param]): key_status, err = param_tol.validate(test_value, bench_value, param) status += key_status if not key_status.passed() and err: msg.append(err) return (comparable, status, "\n".join(msg))