def compare_data(benchmark, test, default_tolerance, tolerances,
ignore_fields=None):
'''Compare two data dictionaries.'''
ignored_params = compat.compat_set(ignore_fields or tuple())
bench_params = compat.compat_set(benchmark) - ignored_params
test_params = compat.compat_set(test) - ignored_params
# Check both the key names and the number of keys in case there are
# different numbers of duplicate keys.
comparable = (bench_params == test_params)
key_counts = dict((key,0) for key in bench_params | test_params)
for (key, val) in benchmark.items():
if key not in ignored_params:
key_counts[key] += len(val)
for (key, val) in test.items():
if key not in ignored_params:
key_counts[key] -= len(val)
comparable = comparable and compat.compat_all(kc == 0 for kc in key_counts.values())
status = Status()
msg = []
if not comparable:
status = Status([False])
bench_only = bench_params - test_params
test_only = test_params - bench_params
msg.append('Different sets of data extracted from benchmark and test.')
if bench_only:
msg.append(" Data only in benchmark: %s." % ", ".join(bench_only))
if test_only:
msg.append(" Data only in test: %s." % ", ".join(test_only))
bench_more = [key for key in key_counts
if key_counts[key] > 0 and key not in bench_only]
test_more = [key for key in key_counts
if key_counts[key] < 0 and key not in test_only]
if bench_more:
msg.append(" More data in benchmark than in test: %s." %
", ".join(bench_more))
if test_more:
msg.append(" More data in test than in benchmark: %s." %
", ".join(test_more))
for param in (bench_params & test_params):
param_tol = tolerances.get(param, default_tolerance)
if param_tol == default_tolerance:
# See if there's a regex that matches.
tol_matches = [tol for tol in tolerances.values()
if tol.name and re.match(tol.name, param)]
if tol_matches:
param_tol = tol_matches[0]
if len(tol_matches) > 1:
warnings.warn('Multiple tolerance regexes match. '
'Using %s.' % (param_tol.name))
for bench_value, test_value in zip(benchmark[param], test[param]):
key_status, err = param_tol.validate(test_value, bench_value, param)
status += key_status
if not key_status.passed() and err:
msg.append(err)
return (comparable, status, "\n".join(msg))
def select_tests(all_tests, test_categories, selected_categories, prefix=''):
'''Return the set of tests contained by the selected test categories.'''
test_categories['_all_'] = [test.path for test in all_tests]
if ('_default_' in selected_categories
and '_default_' not in test_categories):
selected_categories = ['_all_']
# Recursively expand job categories.
while compat.compat_any(cat in test_categories
for cat in selected_categories):
tmp = []
for cat in selected_categories:
if cat in test_categories:
tmp.extend(test_categories[cat])
else:
# cat has been fully expanded and now refers to a test
# contained within the directory named cat.
tmp.append(cat)
selected_categories = tmp
# Select tests to run.
tests = []
parent = lambda pdir, cdir: \
not os.path.relpath(cdir, start=pdir).startswith(os.pardir)
for cat in compat.compat_set(selected_categories):
# test paths are relative to the config directory but absolute paths
# are stored .
found = False
cat_paths = glob.glob(os.path.join(prefix, cat))
for test in all_tests:
if cat == test.name:
found = True
tests.append(test)
elif compat.compat_any(
os.path.exists(path) and os.path.samefile(path, test.path)
for path in cat_paths):
found = True
tests.append(test)
elif compat.compat_any(
parent(path, test.path) for path in cat_paths):
# test contained within a subdirectory of a cat_path.
found = True
tests.append(test)
if not found:
print('WARNING: %s test/category not found.\n' % cat)
# Only want to run each test once.
tests = list(compat.compat_set(tests))
return tests
def select_tests(all_tests, test_categories, selected_categories, prefix=''):
'''Return the set of tests contained by the selected test categories.'''
test_categories['_all_'] = [test.path for test in all_tests]
if ('_default_' in selected_categories
and '_default_' not in test_categories):
selected_categories = ['_all_']
# Recursively expand job categories.
while compat.compat_any(
cat in test_categories for cat in selected_categories
):
tmp = []
for cat in selected_categories:
if cat in test_categories:
tmp.extend(test_categories[cat])
else:
# cat has been fully expanded and now refers to a test
# contained within the directory named cat.
tmp.append(cat)
selected_categories = tmp
# Select tests to run.
tests = []
parent = lambda pdir, cdir: \
not os.path.relpath(cdir, start=pdir).startswith(os.pardir)
for cat in compat.compat_set(selected_categories):
# test paths are relative to the config directory but absolute paths
# are stored .
found = False
cat_paths = glob.glob(os.path.join(prefix, cat))
for test in all_tests:
if cat == test.name:
found = True
tests.append(test)
elif compat.compat_any(os.path.exists(path) and
os.path.samefile(path, test.path) for path in cat_paths):
found = True
tests.append(test)
elif compat.compat_any(parent(path, test.path)
for path in cat_paths):
# test contained within a subdirectory of a cat_path.
found = True
tests.append(test)
if not found:
print('WARNING: %s test/category not found.\n' % cat)
# Only want to run each test once.
tests = list(compat.compat_set(tests))
return tests
def compare_data(benchmark,
test,
default_tolerance,
tolerances,
ignore_fields=None):
'''Compare two data dictionaries.'''
ignored_params = compat.compat_set(ignore_fields or tuple())
bench_params = compat.compat_set(benchmark) - ignored_params
test_params = compat.compat_set(test) - ignored_params
# Check both the key names and the number of keys in case there are
# different numbers of duplicate keys.
comparable = (bench_params == test_params)
key_counts = dict((key, 0) for key in bench_params | test_params)
for (key, val) in benchmark.items():
if key not in ignored_params:
key_counts[key] += len(val)
for (key, val) in test.items():
if key not in ignored_params:
key_counts[key] -= len(val)
comparable = comparable and compat.compat_all(
kc == 0 for kc in key_counts.values())
status = Status()
msg = []
if not comparable:
status = Status([False])
bench_only = bench_params - test_params
test_only = test_params - bench_params
msg.append('Different sets of data extracted from benchmark and test.')
if bench_only:
msg.append(" Data only in benchmark: %s." %
", ".join(bench_only))
if test_only:
msg.append(" Data only in test: %s." % ", ".join(test_only))
bench_more = [
key for key in key_counts
if key_counts[key] > 0 and key not in bench_only
]
test_more = [
key for key in key_counts
if key_counts[key] < 0 and key not in test_only
]
if bench_more:
msg.append(" More data in benchmark than in test: %s." %
", ".join(bench_more))
if test_more:
msg.append(" More data in test than in benchmark: %s." %
", ".join(test_more))
for param in (bench_params & test_params):
param_tol = tolerances.get(param, default_tolerance)
if param_tol == default_tolerance:
# See if there's a regex that matches.
tol_matches = [
tol for tol in tolerances.values()
if tol.name and re.match(tol.name, param)
]
if tol_matches:
param_tol = tol_matches[0]
if len(tol_matches) > 1:
warnings.warn('Multiple tolerance regexes match. '
'Using %s.' % (param_tol.name))
for bench_value, test_value in zip(benchmark[param], test[param]):
key_status, err = param_tol.validate(test_value, bench_value,
param)
status += key_status
if not key_status.passed() and err:
msg.append(err)
return (comparable, status, "\n".join(msg))
